From ac34df9898c8686fa5ff52bfe7a11340056fd4c1 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:34:30 +0200 Subject: [PATCH 01/62] Add modular block authentication schemes --- Cargo.lock | 170 ++++++- README.md | 22 + crates/orchestrator/README.md | 2 +- crates/orchestrator/src/benchmark.rs | 3 +- crates/orchestrator/src/main.rs | 3 +- crates/starfish-core/Cargo.toml | 1 + crates/starfish-core/src/block_handler.rs | 6 +- .../src/bls_certificate_aggregator.rs | 21 +- crates/starfish-core/src/committee.rs | 22 +- crates/starfish-core/src/config.rs | 37 +- crates/starfish-core/src/core.rs | 26 +- crates/starfish-core/src/crypto.rs | 424 ++++++++++++++---- crates/starfish-core/src/dag_state.rs | 103 ++++- crates/starfish-core/src/encoder.rs | 3 +- crates/starfish-core/src/net_sync.rs | 13 +- crates/starfish-core/src/sailfish_service.rs | 1 - crates/starfish-core/src/threshold_clock.rs | 8 +- .../src/transactions_generator.rs | 3 +- crates/starfish-core/src/types.rs | 418 +++++++++++++++-- crates/starfish-core/src/validator.rs | 37 +- crates/starfish/src/main.rs | 9 +- 21 files changed, 1122 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33c4e143..fc80648f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -642,6 +642,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bincode" version = "1.3.3" @@ -741,7 +747,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -929,6 +935,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "color-eyre" version = "0.6.5" @@ -979,6 +991,12 @@ version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1020,6 +1038,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1089,6 +1116,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -1110,6 +1146,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek-ng" version = "4.1.1" @@ -1158,6 +1203,16 @@ dependencies = [ "syn 2.0.115", ] +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.6" @@ -1184,10 +1239,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "crypto-common 0.2.2", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1691,6 +1755,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "ctutils", + "typenum", + "zeroize", +] + [[package]] name = "hyper" version = "0.14.32" @@ -2064,6 +2139,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2296,6 +2381,34 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ml-dsa" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add6b9d92e496f16f4526d68ff29da1483aba4b119baeab8bed3b9e3544a6f3d" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", + "hybrid-array", + "module-lattice", + "pkcs8", + "shake", + "signature", + "zeroize", +] + +[[package]] +name = "module-lattice" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +dependencies = [ + "ctutils", + "hybrid-array", + "num-traits", + "zeroize", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -2537,6 +2650,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -2814,7 +2937,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffef0520d30fbd4151fb20e262947ae47fb0ab276a744a19b6398438105a072" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "fixedbitset", "once_cell", "readme-rustdocifier", @@ -3297,7 +3420,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -3309,10 +3432,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak", + "sponge-cursor", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -3359,6 +3493,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -3400,6 +3543,22 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + [[package]] name = "ssh2" version = "0.9.5" @@ -3452,6 +3611,7 @@ dependencies = [ "libc", "lz4_flex", "memmap2", + "ml-dsa", "parking_lot", "prettytable-rs", "prometheus 0.13.4", diff --git a/README.md b/README.md index 8afdc05e..e4cce090 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,28 @@ achieving 2-round optimistic commit latency. leader, data availability) in block headers, with async verification offloaded from the critical path. +### Starfish block authentication experiments + +Plain Starfish can be run with three interchangeable block-authentication +schemes: + +| CLI name | Block authentication | +|---|---| +| `starfish` | Ed25519 signature | +| `starfish-mac` | Full vector of pairwise keyed-BLAKE3 MAC tags | +| `starfish-ml-dsa-44` | ML-DSA-44 signature | + +For all three variants, `BlockReference.digest` is the BLAKE3 hash of the +canonical block content only. The authentication proof is a separate header +field and does not change the block reference. A `starfish-mac` block carries +exactly one tag for every committee member; each receiver verifies only its +own tag. Benchmark genesis deterministically generates the pairwise MAC keys, +ML-DSA seeds, and public keys in the node configuration. + +This is research/benchmark code. The RustCrypto `ml-dsa` implementation used +here states that it has not been independently audited and should not be +treated as production-ready cryptography. + ## Dissemination Modes Every protocol can run with any of three dissemination strategies diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index f441425c..ba884de6 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -129,7 +129,7 @@ each load generator submits a fixed load of 100 tx/s or more precisely 10 tx every 100ms. Performance measurements are collected by regularly scraping the Prometheus metrics exposed by the load generators. -Available consensus protocols: `starfish`, `starfish-speed`, `sparse-starfish-speed`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. +Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `sparse-starfish-speed`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index 4f8d84f4..6965d4a3 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -55,7 +55,8 @@ pub struct BenchmarkParametersGeneric { /// paying for data sent between the nodes. pub use_internal_ip_address: bool, // Consensus protocol to deploy - // (starfish | starfish-speed | sparse-starfish-speed | starfish-bls | + // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-speed | + // sparse-starfish-speed | starfish-bls | // mysticeti | mysticeti-bls | cordial-miners | bluestreak | sailfish-pp) pub consensus_protocol: String, /// number Byzantine nodes diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index eec38cac..545c63fb 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -135,7 +135,8 @@ pub enum Operation { skip_testbed_configuration: bool, /// Protocols to benchmark in order. Available options: - /// starfish | starfish-speed | sparse-starfish-speed | + /// starfish | starfish-mac | starfish-ml-dsa-44 | + /// starfish-speed | sparse-starfish-speed | /// starfish-bls | mysticeti | mysticeti-bls | /// cordial-miners | bluestreak | sailfish-pp #[clap( diff --git a/crates/starfish-core/Cargo.toml b/crates/starfish-core/Cargo.toml index fc9fd95e..0a6332fc 100644 --- a/crates/starfish-core/Cargo.toml +++ b/crates/starfish-core/Cargo.toml @@ -22,6 +22,7 @@ hex = "0.4.3" libc = "0.2.146" lz4_flex = "0.11" memmap2 = "0.7.0" +ml-dsa = { version = "0.1.1", default-features = false, features = ["alloc", "zeroize"] } parking_lot = "0.12.1" prettytable-rs = "0.10" diff --git a/crates/starfish-core/src/block_handler.rs b/crates/starfish-core/src/block_handler.rs index 001fa0a0..871a381a 100644 --- a/crates/starfish-core/src/block_handler.rs +++ b/crates/starfish-core/src/block_handler.rs @@ -33,8 +33,9 @@ const REAL_BLOCK_HANDLER_TXN_GEN_STEP: usize = 32; const _: () = assert_constants(); #[allow(dead_code)] +#[allow(clippy::manual_is_multiple_of)] const fn assert_constants() { - if !REAL_BLOCK_HANDLER_TXN_SIZE.is_multiple_of(REAL_BLOCK_HANDLER_TXN_GEN_STEP) { + if REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0 { panic!("REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0") } } @@ -264,6 +265,7 @@ impl RealCommitHandler { } impl CommitObserver for RealCommitHandler { + #[allow(clippy::manual_is_multiple_of)] fn handle_commit( &mut self, dag_state: &DagState, @@ -292,7 +294,7 @@ impl CommitObserver for RealCommitHandler { let digest_short = u16::from_le_bytes([self.commit_digest[0], self.commit_digest[1]]) & 0x3FF; self.metrics.commit_digest_latest.set(digest_short as i64); - if commit_index.is_multiple_of(100) { + if commit_index % 100 == 0 { self.metrics.commit_digest.set(digest_short as i64); } diff --git a/crates/starfish-core/src/bls_certificate_aggregator.rs b/crates/starfish-core/src/bls_certificate_aggregator.rs index 865dedcd..9f9fe847 100644 --- a/crates/starfish-core/src/bls_certificate_aggregator.rs +++ b/crates/starfish-core/src/bls_certificate_aggregator.rs @@ -297,18 +297,15 @@ impl BlsCertificateAggregator { } else if let Some(&origin_index) = seen_leader_certs.get(&(*leader_ref, *cert)) { push_task_source(&mut origins[origin_index], source); - } else { - if let Some(task) = self.aggregate_same_message_task( - crypto::bls_leader_message(leader_ref), - cert, - ) { - tasks.push(BlsVerificationTask { - block_index: origins.len(), - ..task - }); - seen_leader_certs.insert((*leader_ref, *cert), origins.len()); - origins.push(TaskOrigin::AggLeader(*leader_ref, *cert, vec![source])); - } + } else if let Some(task) = self + .aggregate_same_message_task(crypto::bls_leader_message(leader_ref), cert) + { + tasks.push(BlsVerificationTask { + block_index: origins.len(), + ..task + }); + seen_leader_certs.insert((*leader_ref, *cert), origins.len()); + origins.push(TaskOrigin::AggLeader(*leader_ref, *cert, vec![source])); } } } diff --git a/crates/starfish-core/src/committee.rs b/crates/starfish-core/src/committee.rs index fe8a3bc8..d6515928 100644 --- a/crates/starfish-core/src/committee.rs +++ b/crates/starfish-core/src/committee.rs @@ -10,7 +10,10 @@ use serde::{Deserialize, Serialize}; use crate::{ config::ImportExport, - crypto::{BlsPublicKey, BlsSigner, PublicKey, Signer, dummy_bls_public_key, dummy_public_key}, + crypto::{ + BlsPublicKey, BlsSigner, MlDsa44PublicKey, MlDsa44Signer, PublicKey, Signer, + dummy_bls_public_key, dummy_ml_dsa_44_public_key, dummy_public_key, + }, data::Data, types::{AuthorityIndex, AuthoritySet, RoundNumber, Stake, VerifiedBlock}, }; @@ -143,6 +146,12 @@ impl Committee { .map(Authority::bls_public_key) } + pub fn get_ml_dsa_44_public_key(&self, authority: AuthorityIndex) -> Option<&MlDsa44PublicKey> { + self.authorities + .get(authority as usize) + .map(Authority::ml_dsa_44_public_key) + } + pub fn known_authority(&self, authority: AuthorityIndex) -> bool { (authority as usize) < self.len() } @@ -206,14 +215,17 @@ impl Committee { pub fn new_for_benchmarks(committee_size: usize) -> Arc { let signers = Signer::new_for_test(committee_size); let bls_signers = BlsSigner::new_for_test(committee_size); + let ml_dsa_signers = MlDsa44Signer::new_for_test(committee_size); Self::new( signers .into_iter() .zip(bls_signers) - .map(|(keypair, bls_keypair)| Authority { + .zip(ml_dsa_signers) + .map(|((keypair, bls_keypair), ml_dsa_keypair)| Authority { stake: 1, public_key: keypair.public_key(), bls_public_key: bls_keypair.public_key(), + ml_dsa_44_public_key: ml_dsa_keypair.public_key(), }) .collect(), ) @@ -225,6 +237,7 @@ pub struct Authority { stake: Stake, public_key: PublicKey, bls_public_key: BlsPublicKey, + ml_dsa_44_public_key: MlDsa44PublicKey, } impl Authority { @@ -233,6 +246,7 @@ impl Authority { stake, public_key: dummy_public_key(), bls_public_key: dummy_bls_public_key(), + ml_dsa_44_public_key: dummy_ml_dsa_44_public_key(), } } @@ -247,6 +261,10 @@ impl Authority { pub fn bls_public_key(&self) -> &BlsPublicKey { &self.bls_public_key } + + pub fn ml_dsa_44_public_key(&self) -> &MlDsa44PublicKey { + &self.ml_dsa_44_public_key + } } impl ImportExport for Committee {} diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 38804fe9..05758057 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -12,7 +12,10 @@ use std::{ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ - crypto::{BlsPublicKey, BlsSigner, Signer, dummy_bls_signer, dummy_signer}, + crypto::{ + BlsPublicKey, BlsSigner, MacKey, MlDsa44Signer, Signer, dummy_bls_signer, + dummy_ml_dsa_44_signer, dummy_signer, mac_keyrings_for_test, + }, types::{AuthorityIndex, PublicKey, RoundNumber}, }; @@ -270,6 +273,8 @@ pub struct NodePrivateConfig { authority: AuthorityIndex, pub keypair: Signer, pub bls_keypair: BlsSigner, + pub ml_dsa_44_keypair: MlDsa44Signer, + pub mac_keys: Vec, pub storage_path: PathBuf, } @@ -279,6 +284,8 @@ impl NodePrivateConfig { authority: index, keypair: dummy_signer(), bls_keypair: dummy_bls_signer(), + ml_dsa_44_keypair: dummy_ml_dsa_44_signer(), + mac_keys: Vec::new(), storage_path: PathBuf::from("storage"), } } @@ -286,20 +293,28 @@ impl NodePrivateConfig { pub fn new_for_benchmarks(working_dir: &Path, committee_size: usize) -> Vec { let signers = Signer::new_for_test(committee_size); let bls_signers = BlsSigner::new_for_test(committee_size); + let ml_dsa_signers = MlDsa44Signer::new_for_test(committee_size); + let mac_keyrings = mac_keyrings_for_test(committee_size); signers .into_iter() .zip(bls_signers) + .zip(ml_dsa_signers) + .zip(mac_keyrings) .enumerate() - .map(|(i, (keypair, bls_keypair))| { - let authority = i as AuthorityIndex; - let path = working_dir.join(NodePrivateConfig::default_storage_path(authority)); - Self { - authority, - keypair, - bls_keypair, - storage_path: path, - } - }) + .map( + |(i, (((keypair, bls_keypair), ml_dsa_44_keypair), mac_keys))| { + let authority = i as AuthorityIndex; + let path = working_dir.join(NodePrivateConfig::default_storage_path(authority)); + Self { + authority, + keypair, + bls_keypair, + ml_dsa_44_keypair, + mac_keys, + storage_path: path, + } + }, + ) .collect() } diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index 2a0311cd..2f0d3eab 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -20,7 +20,7 @@ use crate::{ linearizer::CommittedSubDag, universal_committer::{UniversalCommitter, UniversalCommitterBuilder}, }, - crypto::{self, AsBytes, BlsSignatureBytes, BlsSigner, Signer}, + crypto::{self, AsBytes, BlsSignatureBytes, BlsSigner, MacKey, MlDsa44Signer, Signer}, dag_state::{ ByzantineStrategy, CACHED_ROUNDS, CommitData, ConsensusProtocol, DagState, DataSource, OwnBlockData, @@ -32,9 +32,10 @@ use crate::{ state::RecoveredState, store::Store, types::{ - AuthorityIndex, AuthoritySet, BaseTransaction, BlockReference, BlsAggregateCertificate, - Encoder, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, - RoundNumber, SailfishFields, Shard, VerifiedBlock, + AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, + BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, + ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishFields, Shard, + VerifiedBlock, }, }; @@ -60,6 +61,8 @@ pub struct Core { pub(crate) metrics: Arc, signer: Signer, bls_signer: BlsSigner, + ml_dsa_44_signer: MlDsa44Signer, + mac_keys: Arc>, partial_sig_outbox: Option>, // todo - ugly, probably need to merge syncer and core recovered_committed_blocks: Option>, @@ -185,6 +188,8 @@ impl Core { metrics, signer: private_config.keypair, bls_signer: private_config.bls_keypair, + ml_dsa_44_signer: private_config.ml_dsa_44_keypair, + mac_keys: Arc::new(private_config.mac_keys), partial_sig_outbox, recovered_committed_blocks: Some(committed_blocks), recovered_committed_leaders_count: Some(committed_leaders_count), @@ -206,6 +211,10 @@ impl Core { &self.signer } + pub fn mac_keys(&self) -> Arc> { + self.mac_keys.clone() + } + pub fn get_universal_committer(&self) -> UniversalCommitter { self.committer.clone() } @@ -991,14 +1000,19 @@ impl Core { None }; - let mut block = VerifiedBlock::new_with_signer_and_unprovable( + let authorizer = match self.dag_state.block_authentication_scheme { + BlockAuthenticationScheme::Ed25519 => BlockAuthorizer::Ed25519(&self.signer), + BlockAuthenticationScheme::MacVector => BlockAuthorizer::MacVector(&self.mac_keys), + BlockAuthenticationScheme::MlDsa44 => BlockAuthorizer::MlDsa44(&self.ml_dsa_44_signer), + }; + let mut block = VerifiedBlock::new_with_authorizer_and_unprovable( self.authority, clock_round, block_references, voted_leader_ref, acknowledgment_references.to_vec(), time_ns, - &self.signer, + &authorizer, bls_signer_opt, committee_opt, aggregate_dac_sigs, diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 761b2f68..039b2833 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -5,8 +5,12 @@ use std::fmt; use blst::min_sig as bls; -use ed25519_consensus::Signature; -use rand::{SeedableRng, rngs::StdRng}; +use ml_dsa::{ + Keypair as _, MlDsa44, Signature as MlDsaSignature, Signer as MlDsaSignerTrait, + SigningKey as MlDsaSigningKey, Verifier as MlDsaVerifierTrait, + VerifyingKey as MlDsaVerifyingKey, +}; +use rand::{RngCore, SeedableRng, rngs::StdRng}; use rs_merkle::{Hasher, MerkleProof, MerkleTree}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use zeroize::Zeroize; @@ -15,8 +19,8 @@ use crate::{ committee::Committee, crypto, types::{ - AuthorityIndex, AuthoritySet, BaseTransaction, BlockHeader, BlockReference, RoundNumber, - Shard, TimestampNs, + AuthorityIndex, AuthoritySet, BaseTransaction, BlockReference, RoundNumber, Shard, + TimestampNs, }, }; @@ -73,6 +77,11 @@ pub fn sailfish_novote_digest(round: RoundNumber, leader: AuthorityIndex) -> [u8 pub const SIGNATURE_SIZE: usize = 64; pub const BLOCK_DIGEST_SIZE: usize = 32; +pub const MAC_KEY_SIZE: usize = 32; +pub const MAC_TAG_SIZE: usize = 32; +pub const ML_DSA_44_SEED_SIZE: usize = 32; +pub const ML_DSA_44_PUBLIC_KEY_SIZE: usize = 1_312; +pub const ML_DSA_44_SIGNATURE_SIZE: usize = 2_420; pub const TRANSACTIONS_DIGEST_SIZE: usize = 32; @@ -88,6 +97,23 @@ pub struct PublicKey(ed25519_consensus::VerificationKey); #[derive(Clone, Copy, Eq, Ord, PartialOrd, PartialEq, Hash)] pub struct SignatureBytes([u8; SIGNATURE_SIZE]); +/// A pairwise secret key shared by exactly two validators. +#[derive(Clone, Eq, PartialEq)] +pub struct MacKey([u8; MAC_KEY_SIZE]); + +#[derive(Clone, Copy, Ord, PartialOrd)] +pub struct MacTag([u8; MAC_TAG_SIZE]); + +#[derive(Clone, Eq, PartialEq)] +pub struct MlDsa44SignatureBytes(Box<[u8; ML_DSA_44_SIGNATURE_SIZE]>); + +#[derive(Clone)] +pub struct MlDsa44PublicKey(MlDsaVerifyingKey); + +/// Boxed so moving this wrapper does not copy private key material. +#[derive(Clone)] +pub struct MlDsa44Signer(Box>); + // Box ensures value is not copied in memory when Signer itself is moved around // for better security #[derive(Clone)] @@ -175,13 +201,16 @@ impl TransactionsCommitment { } } impl BlockDigest { + pub fn as_array(&self) -> &[u8; BLOCK_DIGEST_SIZE] { + &self.0 + } + pub fn new_without_transactions( authority: AuthorityIndex, round: RoundNumber, block_references: &[BlockReference], acknowledgment_references: &[BlockReference], meta_creation_time_ns: TimestampNs, - signature: &SignatureBytes, merkle_root: Option, strong_vote: Option, ) -> Self { @@ -191,7 +220,6 @@ impl BlockDigest { block_references, acknowledgment_references, meta_creation_time_ns, - signature, merkle_root, strong_vote, None, @@ -204,13 +232,12 @@ impl BlockDigest { block_references: &[BlockReference], acknowledgment_references: &[BlockReference], meta_creation_time_ns: TimestampNs, - signature: &SignatureBytes, merkle_root: Option, strong_vote: Option, unprovable_certificate: Option<&(BlockReference, bool)>, ) -> Self { let mut hasher = Blake3Hasher::new(); - Self::digest_without_signature( + Self::digest_contents( &mut hasher, authority, round, @@ -221,7 +248,6 @@ impl BlockDigest { strong_vote, ); Self::hash_unprovable_certificate(&mut hasher, unprovable_certificate); - hasher.update(signature.as_bytes()); Self(hasher.finalize().into()) } @@ -231,7 +257,6 @@ impl BlockDigest { block_references: &[BlockReference], acknowledgment_references: &[BlockReference], meta_creation_time_ns: TimestampNs, - signature: &SignatureBytes, transactions_commitment: Option, strong_vote: Option, ) -> Self { @@ -241,7 +266,6 @@ impl BlockDigest { block_references, acknowledgment_references, meta_creation_time_ns, - signature, transactions_commitment, strong_vote, None, @@ -254,13 +278,12 @@ impl BlockDigest { block_references: &[BlockReference], acknowledgment_references: &[BlockReference], meta_creation_time_ns: TimestampNs, - signature: &SignatureBytes, transactions_commitment: Option, strong_vote: Option, unprovable_certificate: Option<&(BlockReference, bool)>, ) -> Self { let mut hasher = Blake3Hasher::new(); - Self::digest_without_signature( + Self::digest_contents( &mut hasher, authority, round, @@ -271,11 +294,10 @@ impl BlockDigest { strong_vote, ); Self::hash_unprovable_certificate(&mut hasher, unprovable_certificate); - hasher.update(signature.as_bytes()); Self(hasher.finalize().into()) } - pub(crate) fn digest_without_signature( + pub(crate) fn digest_contents( hasher: &mut Blake3Hasher, authority: AuthorityIndex, round: RoundNumber, @@ -305,7 +327,7 @@ impl BlockDigest { /// Extend a block digest hasher with the generalized unprovable /// certificate reference + strong/standard flavor flag. Called after - /// `digest_without_signature` and before finalizing. No-op when `None`, + /// `digest_contents` and before finalizing. No-op when `None`, /// preserving backward compatibility. pub(crate) fn hash_unprovable_certificate( hasher: &mut Blake3Hasher, @@ -473,33 +495,234 @@ fn deserialize_fixed_bytes<'de, D: Deserializer<'de>, const N: usize>( } } -impl PublicKey { - pub fn verify_signature_in_block( +impl MacKey { + pub fn compute_tag( &self, - header: &BlockHeader, - transactions_commitment: Option, - ) -> Result<(), ed25519_consensus::Error> { - let signature = Signature::from(header.signature().0); - let acknowledgments = header.acknowledgments(); - let mut hasher = Blake3Hasher::new(); - BlockDigest::digest_without_signature( - &mut hasher, - header.authority(), - header.round(), - header.block_references(), - &acknowledgments, - header.meta_creation_time_ns(), - transactions_commitment, - header.strong_vote(), - ); - BlockDigest::hash_unprovable_certificate( - &mut hasher, - header.unprovable_certificate.as_ref(), - ); - let digest: [u8; BLOCK_DIGEST_SIZE] = hasher.finalize().into(); - self.0.verify(&signature, digest.as_ref()) + author: AuthorityIndex, + recipient: AuthorityIndex, + content_digest: &BlockDigest, + ) -> MacTag { + let mut hasher = Blake3Hasher::new_keyed(&self.0); + hasher.update(&author.to_be_bytes()); + hasher.update(&recipient.to_be_bytes()); + hasher.update(content_digest.as_ref()); + MacTag(hasher.finalize().into()) + } +} + +/// Generate deterministic, symmetric pairwise keyrings for local benchmarks +/// and tests. Entry `keyrings[a][b]` equals `keyrings[b][a]`. +#[allow(clippy::needless_range_loop)] +pub fn mac_keyrings_for_test(n: usize) -> Vec> { + let mut rng = StdRng::seed_from_u64(0x5354_4152_4649_5348); + let mut keyrings = vec![vec![MacKey([0; MAC_KEY_SIZE]); n]; n]; + for author in 0..n { + for recipient in author..n { + let mut bytes = [0; MAC_KEY_SIZE]; + rng.fill_bytes(&mut bytes); + let key = MacKey(bytes); + keyrings[author][recipient] = key.clone(); + keyrings[recipient][author] = key; + } + } + keyrings +} + +impl Drop for MacKey { + fn drop(&mut self) { + self.0.zeroize(); + } +} + +impl fmt::Debug for MacKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("MacKey(REDACTED)") + } +} + +impl Serialize for MacKey { + fn serialize(&self, serializer: S) -> Result { + serialize_fixed_bytes(&self.0, serializer) + } +} + +impl<'de> Deserialize<'de> for MacKey { + fn deserialize>(deserializer: D) -> Result { + deserialize_fixed_bytes::(deserializer, "MAC key").map(Self) } +} + +impl AsBytes for MacTag { + fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl PartialEq for MacTag { + fn eq(&self, other: &Self) -> bool { + blake3::Hash::from_bytes(self.0) == blake3::Hash::from_bytes(other.0) + } +} +impl Eq for MacTag {} + +impl std::hash::Hash for MacTag { + fn hash(&self, state: &mut H) { + std::hash::Hash::hash(&self.0, state); + } +} + +impl AsRef<[u8]> for MacTag { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl fmt::Debug for MacTag { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Mac({})", &hex::encode(&self.0[..4])) + } +} + +impl Serialize for MacTag { + fn serialize(&self, serializer: S) -> Result { + serialize_fixed_bytes(&self.0, serializer) + } +} + +impl<'de> Deserialize<'de> for MacTag { + fn deserialize>(deserializer: D) -> Result { + deserialize_fixed_bytes::(deserializer, "MAC tag").map(Self) + } +} + +impl MlDsa44SignatureBytes { + pub fn from_bytes(bytes: [u8; ML_DSA_44_SIGNATURE_SIZE]) -> Self { + Self(Box::new(bytes)) + } +} + +impl AsRef<[u8]> for MlDsa44SignatureBytes { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl fmt::Debug for MlDsa44SignatureBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "MlDsa44Sig({})", &hex::encode(&self.0[..4])) + } +} + +impl Serialize for MlDsa44SignatureBytes { + fn serialize(&self, serializer: S) -> Result { + serialize_fixed_bytes(self.0.as_ref(), serializer) + } +} + +impl<'de> Deserialize<'de> for MlDsa44SignatureBytes { + fn deserialize>(deserializer: D) -> Result { + deserialize_fixed_bytes::(deserializer, "ML-DSA-44 signature") + .map(Self::from_bytes) + } +} + +impl MlDsa44PublicKey { + pub fn from_bytes(bytes: &[u8; ML_DSA_44_PUBLIC_KEY_SIZE]) -> Self { + let encoded = ml_dsa::EncodedVerifyingKey::::from(*bytes); + Self(MlDsaVerifyingKey::decode(&encoded)) + } + + pub fn to_bytes(&self) -> [u8; ML_DSA_44_PUBLIC_KEY_SIZE] { + self.0.encode().into() + } + + pub fn verify_digest_signature( + &self, + digest: &BlockDigest, + signature: &MlDsa44SignatureBytes, + ) -> Result<(), ml_dsa::signature::Error> { + let signature = MlDsaSignature::::try_from(signature.as_ref())?; + self.0.verify(digest.as_ref(), &signature) + } +} + +impl PartialEq for MlDsa44PublicKey { + fn eq(&self, other: &Self) -> bool { + self.to_bytes() == other.to_bytes() + } +} + +impl Eq for MlDsa44PublicKey {} + +impl fmt::Debug for MlDsa44PublicKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "MlDsa44Pk({})", &hex::encode(&self.to_bytes()[..4])) + } +} + +impl Serialize for MlDsa44PublicKey { + fn serialize(&self, serializer: S) -> Result { + serialize_fixed_bytes(&self.to_bytes(), serializer) + } +} + +impl<'de> Deserialize<'de> for MlDsa44PublicKey { + fn deserialize>(deserializer: D) -> Result { + let bytes = deserialize_fixed_bytes::( + deserializer, + "ML-DSA-44 public key", + )?; + Ok(Self::from_bytes(&bytes)) + } +} + +impl MlDsa44Signer { + pub fn new_for_test(n: usize) -> Vec { + let mut rng = StdRng::seed_from_u64(0x4d4c_4453_4134_3400); + (0..n) + .map(|_| { + let mut bytes = [0; ML_DSA_44_SEED_SIZE]; + rng.fill_bytes(&mut bytes); + let seed = ml_dsa::Seed::from(bytes); + Self(Box::new(MlDsaSigningKey::from_seed(&seed))) + }) + .collect() + } + + pub fn sign_digest(&self, digest: &BlockDigest) -> MlDsa44SignatureBytes { + let signature: MlDsaSignature = self.0.sign(digest.as_ref()); + MlDsa44SignatureBytes::from_bytes(signature.encode().into()) + } + + pub fn public_key(&self) -> MlDsa44PublicKey { + MlDsa44PublicKey(self.0.verifying_key()) + } +} + +impl fmt::Debug for MlDsa44Signer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "MlDsa44Signer(public_key={:?})", self.public_key()) + } +} + +impl Serialize for MlDsa44Signer { + fn serialize(&self, serializer: S) -> Result { + let seed: [u8; ML_DSA_44_SEED_SIZE] = self.0.to_seed().into(); + serialize_fixed_bytes(&seed, serializer) + } +} + +impl<'de> Deserialize<'de> for MlDsa44Signer { + fn deserialize>(deserializer: D) -> Result { + let bytes = + deserialize_fixed_bytes::(deserializer, "ML-DSA-44 seed")?; + let seed = ml_dsa::Seed::from(bytes); + Ok(Self(Box::new(MlDsaSigningKey::from_seed(&seed)))) + } +} + +impl PublicKey { pub fn to_bytes(&self) -> [u8; 32] { let mut bytes = [0u8; 32]; bytes.copy_from_slice(self.0.as_ref()); @@ -530,56 +753,6 @@ impl Signer { .collect() } - pub fn sign_block( - &self, - authority: AuthorityIndex, - round: RoundNumber, - block_references: &[BlockReference], - acknowledgment_references: &[BlockReference], - meta_creation_time_ns: TimestampNs, - transactions_commitment: Option, - strong_vote: Option, - ) -> SignatureBytes { - self.sign_block_with_unprovable( - authority, - round, - block_references, - acknowledgment_references, - meta_creation_time_ns, - transactions_commitment, - strong_vote, - None, - ) - } - - pub fn sign_block_with_unprovable( - &self, - authority: AuthorityIndex, - round: RoundNumber, - block_references: &[BlockReference], - acknowledgment_references: &[BlockReference], - meta_creation_time_ns: TimestampNs, - transactions_commitment: Option, - strong_vote: Option, - unprovable_certificate: Option<&(BlockReference, bool)>, - ) -> SignatureBytes { - let mut hasher = Blake3Hasher::new(); - BlockDigest::digest_without_signature( - &mut hasher, - authority, - round, - block_references, - acknowledgment_references, - meta_creation_time_ns, - transactions_commitment, - strong_vote, - ); - BlockDigest::hash_unprovable_certificate(&mut hasher, unprovable_certificate); - let digest: [u8; BLOCK_DIGEST_SIZE] = hasher.finalize().into(); - let signature = self.0.sign(digest.as_ref()); - SignatureBytes(signature.to_bytes()) - } - /// Sign a pre-computed 32-byte digest. Used for Sailfish++ control /// messages (timeout, no-vote) that don't fit the block-signing schema. pub fn sign_digest(&self, digest: &[u8; 32]) -> SignatureBytes { @@ -744,6 +917,15 @@ pub fn dummy_public_key() -> PublicKey { dummy_signer().public_key() } +pub fn dummy_ml_dsa_44_signer() -> MlDsa44Signer { + let seed = ml_dsa::Seed::from([0; ML_DSA_44_SEED_SIZE]); + MlDsa44Signer(Box::new(MlDsaSigningKey::from_seed(&seed))) +} + +pub fn dummy_ml_dsa_44_public_key() -> MlDsa44PublicKey { + dummy_ml_dsa_44_signer().public_key() +} + // --------------------------------------------------------------------------- // BLS12-381 types (min_sig variant: 96-byte G2 public keys, 48-byte G1 // signatures). @@ -1052,6 +1234,44 @@ mod tests { use super::*; use serde::{Deserialize, Serialize}; + #[test] + fn mac_keyrings_are_symmetric_and_bind_recipient() { + let keyrings = mac_keyrings_for_test(4); + let digest = BlockDigest([7; BLOCK_DIGEST_SIZE]); + let tag = keyrings[1][3].compute_tag(1, 3, &digest); + + assert_eq!(tag, keyrings[3][1].compute_tag(1, 3, &digest)); + assert_ne!(tag, keyrings[3][1].compute_tag(1, 2, &digest)); + assert_ne!(tag, keyrings[3][1].compute_tag(2, 3, &digest)); + } + + #[test] + fn ml_dsa_44_sign_verify_and_serde_roundtrip() { + let signer = MlDsa44Signer::new_for_test(1).pop().unwrap(); + let public_key = signer.public_key(); + let digest = BlockDigest([9; BLOCK_DIGEST_SIZE]); + let signature = signer.sign_digest(&digest); + + assert!( + public_key + .verify_digest_signature(&digest, &signature) + .is_ok() + ); + assert!( + public_key + .verify_digest_signature(&BlockDigest([8; BLOCK_DIGEST_SIZE]), &signature) + .is_err() + ); + + let encoded_key = bincode::serialize(&public_key).unwrap(); + let decoded_key: MlDsa44PublicKey = bincode::deserialize(&encoded_key).unwrap(); + let encoded_signature = bincode::serialize(&signature).unwrap(); + let decoded_signature: MlDsa44SignatureBytes = + bincode::deserialize(&encoded_signature).unwrap(); + assert_eq!(public_key, decoded_key); + assert_eq!(signature, decoded_signature); + } + #[test] fn bls_sign_verify_roundtrip() { let signers = BlsSigner::new_for_test(3); @@ -1111,6 +1331,11 @@ mod tests { bls_signer: BlsSigner, bls_public_key: BlsPublicKey, bls_signature: BlsSignatureBytes, + mac_key: MacKey, + mac_tag: MacTag, + ml_dsa_44_signer: MlDsa44Signer, + ml_dsa_44_public_key: MlDsa44PublicKey, + ml_dsa_44_signature: MlDsa44SignatureBytes, } #[test] @@ -1118,15 +1343,24 @@ mod tests { let signer = Signer::new_for_test(1).pop().unwrap(); let public_key = signer.public_key(); let bls_signer = dummy_bls_signer(); + let mac_key = MacKey([10; MAC_KEY_SIZE]); + let block_digest = BlockDigest([7u8; BLOCK_DIGEST_SIZE]); + let mac_tag = mac_key.compute_tag(0, 1, &block_digest); + let ml_dsa_44_signer = dummy_ml_dsa_44_signer(); let fixture = CryptoYamlFixture { signer, public_key, - block_digest: BlockDigest([7u8; BLOCK_DIGEST_SIZE]), + block_digest, transactions_commitment: TransactionsCommitment([8u8; TRANSACTIONS_DIGEST_SIZE]), signature: SignatureBytes([9u8; SIGNATURE_SIZE]), bls_public_key: bls_signer.public_key(), bls_signature: bls_signer.sign_digest(&[5u8; 32]), bls_signer, + mac_key, + mac_tag, + ml_dsa_44_public_key: ml_dsa_44_signer.public_key(), + ml_dsa_44_signature: ml_dsa_44_signer.sign_digest(&block_digest), + ml_dsa_44_signer, }; let yaml = serde_yaml::to_string(&fixture).unwrap(); @@ -1142,6 +1376,14 @@ mod tests { assert!(fixture.signature == decoded.signature); assert_eq!(fixture.bls_public_key, decoded.bls_public_key); assert_eq!(fixture.bls_signature, decoded.bls_signature); + assert_eq!(fixture.mac_key, decoded.mac_key); + assert_eq!(fixture.mac_tag, decoded.mac_tag); + assert_eq!(fixture.ml_dsa_44_public_key, decoded.ml_dsa_44_public_key); + assert_eq!(fixture.ml_dsa_44_signature, decoded.ml_dsa_44_signature); + assert_eq!( + fixture.ml_dsa_44_signer.public_key(), + decoded.ml_dsa_44_signer.public_key() + ); assert_eq!( fixture.bls_signer.public_key(), decoded.bls_signer.public_key() diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 8f41a3a0..be428fb0 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -32,9 +32,9 @@ use crate::{ store::Store, threshold_clock::ThresholdClockAggregator, types::{ - AuthorityIndex, AuthoritySet, BlockDigest, BlockReference, BlsAggregateCertificate, - ProvableShard, RoundNumber, SailfishNoVoteCert, SailfishTimeoutCert, TransactionData, - VerifiedBlock, + AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, + BlsAggregateCertificate, ProvableShard, RoundNumber, SailfishNoVoteCert, + SailfishTimeoutCert, TransactionData, VerifiedBlock, }, }; @@ -113,7 +113,7 @@ pub enum DacCertificateVerificationState { Rejected, } -#[derive(Clone, Debug, Copy, PartialEq)] +#[derive(Clone, Debug, Copy, Eq, PartialEq)] pub enum ConsensusProtocol { Mysticeti, CordialMiners, @@ -167,19 +167,25 @@ pub enum ConsensusProtocol { impl ConsensusProtocol { pub fn from_str(s: &str) -> Self { + ProtocolConfig::from_str(s) + .unwrap_or_else(|error| panic!("{error}")) + .consensus_protocol + } + + fn from_known_str(s: &str) -> Option { match s { - "mysticeti" => ConsensusProtocol::Mysticeti, - "cordial-miners" => ConsensusProtocol::CordialMiners, - "starfish" => ConsensusProtocol::Starfish, - "starfish-bls" | "starfish-l" => ConsensusProtocol::StarfishBls, - "starfish-speed" | "starfish-s" => ConsensusProtocol::StarfishSpeed, - "sailfish++" | "sailfish-pp" => ConsensusProtocol::SailfishPlusPlus, - "bluestreak" => ConsensusProtocol::Bluestreak, - "mysticeti-bls" | "mysticeti-l" => ConsensusProtocol::MysticetiBls, + "mysticeti" => Some(ConsensusProtocol::Mysticeti), + "cordial-miners" => Some(ConsensusProtocol::CordialMiners), + "starfish" => Some(ConsensusProtocol::Starfish), + "starfish-bls" | "starfish-l" => Some(ConsensusProtocol::StarfishBls), + "starfish-speed" | "starfish-s" => Some(ConsensusProtocol::StarfishSpeed), + "sailfish++" | "sailfish-pp" => Some(ConsensusProtocol::SailfishPlusPlus), + "bluestreak" => Some(ConsensusProtocol::Bluestreak), + "mysticeti-bls" | "mysticeti-l" => Some(ConsensusProtocol::MysticetiBls), "sparse-starfish-speed" | "sparse-starfish" | "ssfs" => { - ConsensusProtocol::SparseStarfishSpeed + Some(ConsensusProtocol::SparseStarfishSpeed) } - _ => ConsensusProtocol::Starfish, + _ => None, } } @@ -289,6 +295,36 @@ impl ConsensusProtocol { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProtocolConfig { + pub consensus_protocol: ConsensusProtocol, + pub block_authentication_scheme: BlockAuthenticationScheme, +} + +impl ProtocolConfig { + pub fn from_str(value: &str) -> Result { + let (consensus_protocol, block_authentication_scheme) = match value { + "starfish-mac" => ( + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + ), + "starfish-ml-dsa-44" => ( + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MlDsa44, + ), + known => ( + ConsensusProtocol::from_known_str(known) + .ok_or_else(|| format!("Unknown consensus protocol '{known}'"))?, + BlockAuthenticationScheme::Ed25519, + ), + }; + Ok(Self { + consensus_protocol, + block_authentication_scheme, + }) + } +} + const STARFISH_SPEED_HINT_WINDOW_LEADER_ROUNDS: usize = 10; #[allow(unused)] @@ -338,6 +374,7 @@ pub struct DagState { store: Arc, metrics: Arc, pub(crate) consensus_protocol: ConsensusProtocol, + pub(crate) block_authentication_scheme: BlockAuthenticationScheme, pub(crate) committee_size: usize, pub(crate) byzantine_strategy: Option, committee: Arc, @@ -542,7 +579,8 @@ impl DagState { Arc::new(RocksStore::open(&path).expect("Failed to open RocksDB")) } }; - let consensus_protocol = ConsensusProtocol::from_str(&consensus); + let protocol_config = ProtocolConfig::from_str(&consensus).expect("validated protocol"); + let consensus_protocol = protocol_config.consensus_protocol; let resolved_dissemination = consensus_protocol.resolve_dissemination_mode(dissemination_mode); let push_mode = matches!( @@ -829,6 +867,7 @@ impl DagState { dag_state_inner: Arc::new(RwLock::new(inner)), metrics, consensus_protocol, + block_authentication_scheme: protocol_config.block_authentication_scheme, round_block_cache: Arc::new(parking_lot::Mutex::new(AHashMap::new())), genesis, strong_vote_adaptive_acknowledgments, @@ -3400,7 +3439,7 @@ mod tests { use super::{ ByzantineStrategy, CACHED_ROUNDS, CertificateEvent, ConsensusProtocol, - DacCertificateVerificationState, DagState, DataSource, OwnBlockData, + DacCertificateVerificationState, DagState, DataSource, OwnBlockData, ProtocolConfig, }; use crate::{ committee::Committee, @@ -3412,9 +3451,9 @@ mod tests { data::Data, metrics::Metrics, types::{ - AuthorityIndex, AuthoritySet, BaseTransaction, BlockReference, BlsAggregateCertificate, - ProvableShard, RoundNumber, SailfishFields, SailfishNoVoteCert, Transaction, - VerifiedBlock, + AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, + BlockReference, BlsAggregateCertificate, ProvableShard, RoundNumber, SailfishFields, + SailfishNoVoteCert, Transaction, VerifiedBlock, }, }; @@ -4762,4 +4801,30 @@ mod tests { DisseminationMode::PushCausal ); } + + #[test] + fn protocol_config_selects_starfish_block_authentication() { + assert_eq!( + ProtocolConfig::from_str("starfish").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Starfish, + block_authentication_scheme: BlockAuthenticationScheme::Ed25519, + } + ); + assert_eq!( + ProtocolConfig::from_str("starfish-mac").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Starfish, + block_authentication_scheme: BlockAuthenticationScheme::MacVector, + } + ); + assert_eq!( + ProtocolConfig::from_str("starfish-ml-dsa-44").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Starfish, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, + } + ); + assert!(ProtocolConfig::from_str("starfish-unknown").is_err()); + } } diff --git a/crates/starfish-core/src/encoder.rs b/crates/starfish-core/src/encoder.rs index 122b2e48..a1e0bc01 100644 --- a/crates/starfish-core/src/encoder.rs +++ b/crates/starfish-core/src/encoder.rs @@ -42,6 +42,7 @@ impl ShardEncoder for Encoder { data } + #[allow(clippy::manual_is_multiple_of)] fn encode_transactions( &mut self, block: &[BaseTransaction], @@ -57,7 +58,7 @@ impl ShardEncoder for Encoder { let mut shard_bytes = (bytes_length + 4).div_ceil(info_length); // Ensure shard_bytes meets alignment requirements (must be multiple of 2). - if !shard_bytes.is_multiple_of(2) { + if shard_bytes % 2 != 0 { shard_bytes += 1; } diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index cef8dd42..3b95bf62 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -34,7 +34,7 @@ use crate::{ }, core::Core, core_thread::CoreThreadDispatcher, - crypto::BlsSigner, + crypto::{BlsSigner, MacKey}, dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, metrics::{Metrics, UtilizationTimerVecExt}, @@ -435,12 +435,14 @@ fn spawn_header_worker( } let mut block: VerifiedBlock = (*data_block).clone(); tracing::debug!("Received {} from {}", block, peer); - match block.verify( + match block.verify_with_authentication( &inner.committee, own_id as usize, peer_id as usize, &mut encoder, consensus_protocol, + inner.dag_state.block_authentication_scheme, + &inner.mac_keys, ) { Ok(shard) => { debug_assert!(shard.is_none(), "shard must be None for header-only blocks") @@ -975,12 +977,14 @@ impl ConnectionHandler shard, Err(e) => { @@ -1329,6 +1333,7 @@ pub struct NetworkSyncerInner { pub block_ready_notify: Arc, pub proposal_round_notify: Arc, pub committee: Arc, + pub mac_keys: Arc>, pub dissemination_mode: DisseminationMode, pub causal_push_shard_round_lag: RoundNumber, stop: mpsc::Sender<()>, @@ -1367,6 +1372,7 @@ impl NetworkSyncer let (committed, committed_leaders_count) = core.take_recovered_committed(); commit_observer.recover_committed(committed, committed_leaders_count); let committee = core.committee().clone(); + let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); let dissemination_mode = dag_state .consensus_protocol @@ -1459,6 +1465,7 @@ impl NetworkSyncer syncer, proposal_round_notify, committee, + mac_keys, dissemination_mode, causal_push_shard_round_lag: node_parameters.causal_push_shard_round_lag, stop: stop_sender.clone(), diff --git a/crates/starfish-core/src/sailfish_service.rs b/crates/starfish-core/src/sailfish_service.rs index 62fb3d0e..b580fac9 100644 --- a/crates/starfish-core/src/sailfish_service.rs +++ b/crates/starfish-core/src/sailfish_service.rs @@ -551,7 +551,6 @@ mod tests { &[], &[], 1, - &crate::crypto::SignatureBytes::default(), None, None, ), diff --git a/crates/starfish-core/src/threshold_clock.rs b/crates/starfish-core/src/threshold_clock.rs index b70bf4c0..7ba883b0 100644 --- a/crates/starfish-core/src/threshold_clock.rs +++ b/crates/starfish-core/src/threshold_clock.rs @@ -75,10 +75,7 @@ impl ThresholdClockAggregator { mod tests { use super::*; - use crate::{ - crypto::SignatureBytes, - types::{AckFields, AuthorityIndex, BlockDigest, RoundNumber}, - }; + use crate::types::{AckFields, AuthorityIndex, BlockAuthentication, BlockDigest, RoundNumber}; fn make_header( authority: AuthorityIndex, @@ -100,14 +97,13 @@ mod tests { &block_references, &ack_refs, 0, - &SignatureBytes::default(), None, None, ), }, block_references, meta_creation_time_ns: 0, - signature: SignatureBytes::default(), + authentication: BlockAuthentication::None, transactions_commitment: None, ack: Some(AckFields { intersection: None, diff --git a/crates/starfish-core/src/transactions_generator.rs b/crates/starfish-core/src/transactions_generator.rs index 0ca474f9..3725fe6d 100644 --- a/crates/starfish-core/src/transactions_generator.rs +++ b/crates/starfish-core/src/transactions_generator.rs @@ -59,6 +59,7 @@ impl TransactionGenerator { ); } + #[allow(clippy::manual_is_multiple_of)] pub async fn run(mut self) { let load = self.parameters.load; let max_transactions_per_block_interval = load.div_ceil(Self::BATCHES_IN_SECOND); @@ -182,7 +183,7 @@ impl TransactionGenerator { return; } - if counter.is_multiple_of(10_000) { + if counter % 10_000 == 0 { self.metrics .submitted_transactions_bytes .inc_by(tx_to_report * tx_size as u64); diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 53cbc6e6..cb861f00 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -41,8 +41,8 @@ use crate::{ committee::Committee, crypto, crypto::{ - AsBytes, BlsSignatureBytes, BlsSigner, CryptoHash, SignatureBytes, Signer, - TransactionsCommitment, + AsBytes, BlsSignatureBytes, BlsSigner, CryptoHash, MacKey, MacTag, MlDsa44SignatureBytes, + MlDsa44Signer, SignatureBytes, Signer, TransactionsCommitment, }, dag_state::ConsensusProtocol, data::{Data, IN_MEMORY_BLOCKS, IN_MEMORY_BLOCKS_BYTES}, @@ -87,9 +87,7 @@ impl PartialOrd for BlockReference { } // --------------------------------------------------------------------------- -// BlockHeader — signed, content-addressed block identity. -// Contains exactly the fields that feed into BlockDigest::new() and -// sign_block(). +// BlockHeader — authenticated, content-addressed block identity. // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- @@ -239,7 +237,7 @@ pub struct SailfishNoVoteCert { } /// Protocol-specific fields embedded in SailfishPlusPlus block headers. -/// Part of the signed block hash. +/// Part of the authenticated block content hash. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct SailfishFields { /// Timeout certificate for the previous round, if this block advances @@ -252,11 +250,56 @@ pub struct SailfishFields { } // --------------------------------------------------------------------------- -// BlockHeader — signed, content-addressed block identity. -// Contains exactly the fields that feed into BlockDigest::new() and -// sign_block(). +// BlockHeader — authenticated, content-addressed block identity. // --------------------------------------------------------------------------- +#[derive(Clone, Eq, PartialEq, Serialize, Deserialize, Debug)] +pub enum BlockAuthentication { + /// Only valid for locally constructed genesis blocks. + None, + Ed25519(SignatureBytes), + MacVector(Vec), + MlDsa44(MlDsa44SignatureBytes), +} + +#[derive(Clone, Copy, Eq, PartialEq, Serialize, Deserialize, Debug)] +pub enum BlockAuthenticationScheme { + Ed25519, + MacVector, + MlDsa44, +} + +pub enum BlockAuthorizer<'a> { + Ed25519(&'a Signer), + MacVector(&'a [MacKey]), + MlDsa44(&'a MlDsa44Signer), +} + +impl BlockAuthorizer<'_> { + fn authenticate( + &self, + author: AuthorityIndex, + content_digest: &BlockDigest, + ) -> BlockAuthentication { + match self { + Self::Ed25519(signer) => { + BlockAuthentication::Ed25519(signer.sign_digest(content_digest.as_array())) + } + Self::MacVector(keys) => BlockAuthentication::MacVector( + keys.iter() + .enumerate() + .map(|(recipient, key)| { + key.compute_tag(author, recipient as AuthorityIndex, content_digest) + }) + .collect(), + ), + Self::MlDsa44(signer) => { + BlockAuthentication::MlDsa44(signer.sign_digest(content_digest)) + } + } + } +} + #[derive(Clone, Serialize, Deserialize)] pub struct BlockHeader { // -- Base fields (all protocols) ------------------------------------------ @@ -267,8 +310,9 @@ pub struct BlockHeader { pub(crate) block_references: Vec, /// Creation time as reported by creator (currently not enforced). pub(crate) meta_creation_time_ns: TimestampNs, - /// Signature by the block author over the header fields. - pub(crate) signature: SignatureBytes, + /// Authentication proof over `reference.digest`. This field is not part of + /// the content-addressed block identity. + pub(crate) authentication: BlockAuthentication, /// Explicit payload commitment stored in the header. /// Starfish-family protocols carry the Merkle root over encoded shards. /// Full-block protocols leave this as `None` and recompute the raw @@ -367,8 +411,8 @@ impl BlockHeader { self.reference.author_round() } - pub fn signature(&self) -> &SignatureBytes { - &self.signature + pub fn authentication(&self) -> &BlockAuthentication { + &self.authentication } pub fn meta_creation_time_ns(&self) -> TimestampNs { @@ -777,7 +821,6 @@ impl VerifiedBlock { &block_references, &acknowledgments, meta_creation_time_ns, - &signature, merkle_root, strong_vote, unprovable_certificate.as_ref(), @@ -785,7 +828,7 @@ impl VerifiedBlock { }, block_references, meta_creation_time_ns, - signature, + authentication: BlockAuthentication::Ed25519(signature), transactions_commitment: merkle_root, ack: Some(AckFields { intersection: acknowledgment_intersection, @@ -822,14 +865,13 @@ impl VerifiedBlock { &block_refs, &ack_refs, 0, - &SignatureBytes::default(), None, None, ), }, block_references: block_refs, meta_creation_time_ns: 0, - signature: SignatureBytes::default(), + authentication: BlockAuthentication::None, transactions_commitment: None, ack: None, strong_vote: None, @@ -915,6 +957,54 @@ impl VerifiedBlock { precomputed_leader_sig: Option, sailfish: Option, unprovable_certificate: Option<(BlockReference, bool)>, + ) -> Self { + let authorizer = BlockAuthorizer::Ed25519(signer); + Self::new_with_authorizer_and_unprovable( + authority, + round, + block_references, + voted_leader_ref, + acknowledgment_references, + meta_creation_time_ns, + &authorizer, + bls_signer, + committee_opt, + aggregate_dac_sigs, + transactions, + encoded_transactions, + consensus_protocol, + strong_vote, + aggregate_round_sig, + certified_leader, + precomputed_round_sig, + precomputed_leader_sig, + sailfish, + unprovable_certificate, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn new_with_authorizer_and_unprovable( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + voted_leader_ref: Option, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + authorizer: &BlockAuthorizer<'_>, + bls_signer: Option<&BlsSigner>, + committee_opt: Option<&Committee>, + aggregate_dac_sigs: Vec, + transactions: Vec, + encoded_transactions: Option>, + consensus_protocol: ConsensusProtocol, + strong_vote: Option, + aggregate_round_sig: Option, + certified_leader: Option<(BlockReference, BlsAggregateCertificate)>, + precomputed_round_sig: Option, + precomputed_leader_sig: Option, + sailfish: Option, + unprovable_certificate: Option<(BlockReference, bool)>, ) -> Self { let supports_acknowledgments = consensus_protocol.supports_acknowledgments(); let header_transactions_commitment = if consensus_protocol.supports_acknowledgments() { @@ -950,7 +1040,7 @@ impl VerifiedBlock { &acknowledgment_references, aggregate_dac_sigs, ); - let signature = signer.sign_block_with_unprovable( + let content_digest = BlockDigest::new_without_transactions_with_unprovable( authority, round, &block_references, @@ -960,6 +1050,7 @@ impl VerifiedBlock { strong_vote, unprovable_certificate.as_ref(), ); + let authentication = authorizer.authenticate(authority, &content_digest); // Build BLS fields when the StarfishBls path is active. Partial round // and leader signatures are embedded as belt-and-suspenders alongside @@ -996,21 +1087,11 @@ impl VerifiedBlock { reference: BlockReference { authority, round, - digest: BlockDigest::new_without_transactions_with_unprovable( - authority, - round, - &block_references, - &acknowledgments, - meta_creation_time_ns, - &signature, - digest_transactions_commitment, - strong_vote, - unprovable_certificate.as_ref(), - ), + digest: content_digest, }, block_references, meta_creation_time_ns, - signature, + authentication, transactions_commitment: header_transactions_commitment, ack: supports_acknowledgments.then_some(AckFields { intersection: acknowledgment_intersection, @@ -1075,8 +1156,8 @@ impl VerifiedBlock { self.header.author_round() } - pub fn signature(&self) -> &SignatureBytes { - self.header.signature() + pub fn authentication(&self) -> &BlockAuthentication { + self.header.authentication() } pub fn meta_creation_time_ns(&self) -> TimestampNs { @@ -1208,12 +1289,36 @@ impl VerifiedBlock { _peer_id: usize, encoder: &mut Encoder, consensus_protocol: ConsensusProtocol, + ) -> eyre::Result> { + self.verify_with_authentication( + committee, + own_id, + _peer_id, + encoder, + consensus_protocol, + BlockAuthenticationScheme::Ed25519, + &[], + ) + } + + pub fn verify_with_authentication( + &mut self, + committee: &Committee, + own_id: usize, + _peer_id: usize, + encoder: &mut Encoder, + consensus_protocol: ConsensusProtocol, + authentication_scheme: BlockAuthenticationScheme, + mac_keys: &[MacKey], ) -> eyre::Result> { let (shard, digest_transactions_commitment) = self.verify_transactions(committee, own_id, encoder, consensus_protocol)?; self.verify_block_structure( committee, + own_id, consensus_protocol, + authentication_scheme, + mac_keys, digest_transactions_commitment, )?; Ok(shard) @@ -1280,11 +1385,14 @@ impl VerifiedBlock { } } - /// Verify digest, signature, includes, and threshold clock. + /// Verify content digest, authentication, includes, and threshold clock. fn verify_block_structure( &self, committee: &Committee, + own_id: usize, consensus_protocol: ConsensusProtocol, + authentication_scheme: BlockAuthenticationScheme, + mac_keys: &[MacKey], digest_transactions_commitment: Option, ) -> eyre::Result<()> { let round = self.round(); @@ -1313,7 +1421,6 @@ impl VerifiedBlock { &self.header.block_references, &acknowledgments, self.header.meta_creation_time_ns, - &self.header.signature, digest_transactions_commitment, self.header.strong_vote, self.header.unprovable_certificate.as_ref(), @@ -1324,17 +1431,57 @@ impl VerifiedBlock { digest, self.digest() ); - let pub_key = committee.get_public_key(self.authority()); - let Some(pub_key) = pub_key else { - bail!("Unknown block author {}", self.authority()) - }; if round == GENESIS_ROUND { bail!("Genesis block should not go through verification"); } - if let Err(e) = - pub_key.verify_signature_in_block(&self.header, digest_transactions_commitment) - { - bail!("Block signature verification has failed: {:?}", e); + match (authentication_scheme, &self.header.authentication) { + (BlockAuthenticationScheme::Ed25519, BlockAuthentication::Ed25519(signature)) => { + let Some(public_key) = committee.get_public_key(self.authority()) else { + bail!("Unknown block author {}", self.authority()) + }; + if let Err(error) = public_key.verify_digest_signature(digest.as_array(), signature) + { + bail!("Block Ed25519 verification has failed: {error:?}"); + } + } + (BlockAuthenticationScheme::MacVector, BlockAuthentication::MacVector(tags)) => { + ensure!( + tags.len() == committee.len(), + "MAC vector length {} does not match committee size {}", + tags.len(), + committee.len(), + ); + ensure!( + own_id < committee.len(), + "Own authority index is out of bounds" + ); + ensure!( + mac_keys.len() == committee.len(), + "MAC keyring length {} does not match committee size {}", + mac_keys.len(), + committee.len(), + ); + let author = self.authority() as usize; + let Some(key) = mac_keys.get(author) else { + bail!("Unknown block author {}", self.authority()) + }; + let expected = key.compute_tag(self.authority(), own_id as AuthorityIndex, &digest); + ensure!( + tags[own_id] == expected, + "Block MAC verification has failed" + ); + } + (BlockAuthenticationScheme::MlDsa44, BlockAuthentication::MlDsa44(signature)) => { + let Some(public_key) = committee.get_ml_dsa_44_public_key(self.authority()) else { + bail!("Unknown block author {}", self.authority()) + }; + if let Err(error) = public_key.verify_digest_signature(&digest, signature) { + bail!("Block ML-DSA-44 verification has failed: {error:?}"); + } + } + (expected, actual) => { + bail!("Expected {expected:?} block authentication, received {actual:?}") + } } for include in &self.header.block_references { ensure!( @@ -2147,6 +2294,191 @@ impl std::hash::Hash for VerifiedBlock { mod tests { use super::*; + fn make_authenticated_starfish_block( + committee: &Committee, + authorizer: &BlockAuthorizer<'_>, + ) -> VerifiedBlock { + let authority = 0; + let round = 1; + let transactions = Vec::new(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded_transactions = encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + VerifiedBlock::new_with_authorizer_and_unprovable( + authority, + round, + committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(), + None, + Vec::new(), + 0, + authorizer, + None, + None, + Vec::new(), + transactions, + Some(encoded_transactions), + ConsensusProtocol::Starfish, + None, + None, + None, + None, + None, + None, + None, + ) + } + + #[test] + fn block_reference_depends_only_on_content_across_authentication_schemes() { + let committee = Committee::new_for_benchmarks(4); + let ed_signers = Signer::new_for_test(committee.len()); + let ml_dsa_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); + let mac_keyrings = crypto::mac_keyrings_for_test(committee.len()); + let ed = BlockAuthorizer::Ed25519(&ed_signers[0]); + let mac = BlockAuthorizer::MacVector(&mac_keyrings[0]); + let ml_dsa = BlockAuthorizer::MlDsa44(&ml_dsa_signers[0]); + + let ed_block = make_authenticated_starfish_block(&committee, &ed); + let mac_block = make_authenticated_starfish_block(&committee, &mac); + let ml_dsa_block = make_authenticated_starfish_block(&committee, &ml_dsa); + + assert_eq!(ed_block.reference(), mac_block.reference()); + assert_eq!(ed_block.reference(), ml_dsa_block.reference()); + assert_ne!(ed_block.authentication(), mac_block.authentication()); + assert_ne!(ed_block.authentication(), ml_dsa_block.authentication()); + } + + #[test] + fn all_authentication_schemes_verify_for_starfish() { + let committee = Committee::new_for_benchmarks(4); + let ed_signers = Signer::new_for_test(committee.len()); + let ml_dsa_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); + let mac_keyrings = crypto::mac_keyrings_for_test(committee.len()); + + let cases = [ + ( + make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::Ed25519(&ed_signers[0]), + ), + BlockAuthenticationScheme::Ed25519, + ), + ( + make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&mac_keyrings[0]), + ), + BlockAuthenticationScheme::MacVector, + ), + ( + make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MlDsa44(&ml_dsa_signers[0]), + ), + BlockAuthenticationScheme::MlDsa44, + ), + ]; + + for (block, scheme) in cases { + for (receiver, receiver_keys) in mac_keyrings.iter().enumerate() { + let mut received = block.clone(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let mac_keys = if scheme == BlockAuthenticationScheme::MacVector { + receiver_keys.as_slice() + } else { + &[] + }; + received + .verify_with_authentication( + &committee, + receiver, + 0, + &mut encoder, + ConsensusProtocol::Starfish, + scheme, + mac_keys, + ) + .unwrap(); + } + } + } + + #[test] + fn rejects_incomplete_mac_vector_and_wrong_authentication_scheme() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let mut mac_block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + let BlockAuthentication::MacVector(tags) = &mut mac_block.header.authentication else { + panic!("expected MAC vector") + }; + tags.pop(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + assert!( + mac_block + .verify_with_authentication( + &committee, + 1, + 0, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[1], + ) + .is_err() + ); + + let mut wrong_tag_block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + let BlockAuthentication::MacVector(tags) = &mut wrong_tag_block.header.authentication + else { + panic!("expected MAC vector") + }; + tags.swap(1, 2); + assert!( + wrong_tag_block + .verify_with_authentication( + &committee, + 1, + 0, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[1], + ) + .is_err() + ); + + let ed_signers = Signer::new_for_test(committee.len()); + let mut ed_block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::Ed25519(&ed_signers[0]), + ); + assert!( + ed_block + .verify_with_authentication( + &committee, + 1, + 0, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MlDsa44, + &[], + ) + .is_err() + ); + } + fn single_signer_cert( digest: [u8; 32], signer: AuthorityIndex, @@ -2368,7 +2700,7 @@ mod tests { reference: BlockReference::new_test(0, 2), block_references: vec![a], meta_creation_time_ns: 0, - signature: SignatureBytes::default(), + authentication: BlockAuthentication::None, transactions_commitment: None, ack: Some(AckFields { intersection: None, diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 252edf9d..420acca5 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -18,14 +18,14 @@ use crate::{ committee::Committee, config::{NodePrivateConfig, NodePublicConfig, Parameters}, core::Core, - dag_state::{ConsensusProtocol, DagState}, + dag_state::{DagState, ProtocolConfig}, metrics::{MetricReporter, Metrics}, net_sync::NetworkSyncer, network::Network, prometheus, runtime::{JoinError, JoinHandle}, transactions_generator::TransactionGenerator, - types::{AuthorityIndex, PartialSig}, + types::{AuthorityIndex, BlockAuthenticationScheme, PartialSig}, }; pub struct Validator { @@ -45,6 +45,35 @@ impl Validator { byzantine_strategy: String, consensus: String, ) -> Result { + let protocol_config = ProtocolConfig::from_str(&consensus).map_err(|error| eyre!(error))?; + match protocol_config.block_authentication_scheme { + BlockAuthenticationScheme::Ed25519 => { + if committee.get_public_key(authority) != Some(&private_config.keypair.public_key()) + { + return Err(eyre!( + "Ed25519 private key does not match committee authority {authority}" + )); + } + } + BlockAuthenticationScheme::MacVector => { + if private_config.mac_keys.len() != committee.len() { + return Err(eyre!( + "MAC keyring length {} does not match committee size {}", + private_config.mac_keys.len(), + committee.len(), + )); + } + } + BlockAuthenticationScheme::MlDsa44 => { + if committee.get_ml_dsa_44_public_key(authority) + != Some(&private_config.ml_dsa_44_keypair.public_key()) + { + return Err(eyre!( + "ML-DSA-44 private key does not match committee authority {authority}" + )); + } + } + } // Network and metrics setup remains the same let network_address = public_config .network_address(authority) @@ -76,7 +105,7 @@ impl Validator { .register(Box::new(pc)) .wrap_err("Failed to register ProcessCollector")?; } - let protocol = ConsensusProtocol::from_str(&consensus); + let protocol = protocol_config.consensus_protocol; let resolved_dissemination = protocol.resolve_dissemination_mode(public_config.parameters.dissemination_mode); let dissemination_str = resolved_dissemination.to_string(); @@ -312,6 +341,8 @@ mod smoke_tests { #[test_case("mysticeti", 0)] #[test_case("cordial-miners", 40)] #[test_case("starfish", 60)] + #[test_case("starfish-mac", 700)] + #[test_case("starfish-ml-dsa-44", 720)] #[test_case("starfish-speed", 80)] #[test_case("starfish-bls", 100)] #[test_case("sailfish++", 120)] diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 2c8e24e5..9c8c6954 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -65,6 +65,8 @@ enum Operation { parameters_path: String, #[clap(long, value_name = "STRING", default_value = "")] byzantine_strategy: String, + /// Consensus/authentication variant (for example `starfish`, + /// `starfish-mac`, or `starfish-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, }, @@ -93,6 +95,8 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, + /// Consensus/authentication variant (for example `starfish`, + /// `starfish-mac`, or `starfish-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, /// Directory to store validator data (default: current directory) @@ -143,6 +147,8 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, + /// Consensus/authentication variant (for example `starfish`, + /// `starfish-mac`, or `starfish-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, #[clap(long, value_name = "INT", default_value_t = 600)] @@ -322,6 +328,7 @@ fn benchmark_genesis( Ok(()) } +#[allow(clippy::manual_is_multiple_of)] async fn local_benchmark( committee_size: usize, mut load: usize, @@ -422,7 +429,7 @@ async fn local_benchmark( )); } } - let is_byzantine = authority.is_multiple_of(3) && authority / 3 < num_byzantine_nodes; + let is_byzantine = authority % 3 == 0 && authority / 3 < num_byzantine_nodes; let validator = if is_byzantine { Validator::start( authority as AuthorityIndex, From fcd39959d22db796a89aa9910f939c397294e937 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:38:10 +0200 Subject: [PATCH 02/62] Flatten MAC vector wire encoding --- crates/starfish-core/src/crypto.rs | 6 ++ crates/starfish-core/src/types.rs | 105 ++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 039b2833..43ddaa25 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -558,6 +558,12 @@ impl AsBytes for MacTag { } } +impl MacTag { + pub(crate) fn from_bytes(bytes: [u8; MAC_TAG_SIZE]) -> Self { + Self(bytes) + } +} + impl PartialEq for MacTag { fn eq(&self, other: &Self) -> bool { blake3::Hash::from_bytes(self.0) == blake3::Hash::from_bytes(other.0) diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index cb861f00..53177a24 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -35,7 +35,7 @@ use ahash::AHashSet; use bytes::Bytes; use eyre::{bail, ensure}; use reed_solomon_simd::{ReedSolomonDecoder, ReedSolomonEncoder}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use crate::{ committee::Committee, @@ -258,10 +258,73 @@ pub enum BlockAuthentication { /// Only valid for locally constructed genesis blocks. None, Ed25519(SignatureBytes), - MacVector(Vec), + MacVector(#[serde(with = "flat_mac_vector")] Vec), MlDsa44(MlDsa44SignatureBytes), } +mod flat_mac_vector { + use super::*; + + pub fn serialize(tags: &[MacTag], serializer: S) -> Result { + if serializer.is_human_readable() { + return tags.serialize(serializer); + } + + let mut bytes = Vec::with_capacity(tags.len() * crypto::MAC_TAG_SIZE); + for tag in tags { + bytes.extend_from_slice(tag.as_ref()); + } + serializer.serialize_bytes(&bytes) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + if deserializer.is_human_readable() { + return Vec::::deserialize(deserializer); + } + + deserializer.deserialize_bytes(FlatMacVectorVisitor) + } + + struct FlatMacVectorVisitor; + + impl<'de> de::Visitor<'de> for FlatMacVectorVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "a flat byte string containing 32 bytes per MAC tag" + ) + } + + #[allow(clippy::manual_is_multiple_of)] + fn visit_bytes(self, bytes: &[u8]) -> Result { + if bytes.len() % crypto::MAC_TAG_SIZE != 0 { + return Err(E::custom(format!( + "invalid flat MAC vector length {}; expected a multiple of {}", + bytes.len(), + crypto::MAC_TAG_SIZE + ))); + } + + Ok(bytes + .chunks_exact(crypto::MAC_TAG_SIZE) + .map(|chunk| { + let mut tag = [0; crypto::MAC_TAG_SIZE]; + tag.copy_from_slice(chunk); + MacTag::from_bytes(tag) + }) + .collect()) + } + + fn visit_byte_buf(self, bytes: Vec) -> Result { + self.visit_bytes(&bytes) + } + } +} + #[derive(Clone, Copy, Eq, PartialEq, Serialize, Deserialize, Debug)] pub enum BlockAuthenticationScheme { Ed25519, @@ -2409,6 +2472,44 @@ mod tests { } } + #[test] + fn mac_vector_uses_flat_binary_encoding() { + let committee = Committee::new_for_benchmarks(10); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + + let encoded = bincode::serialize(block.authentication()).unwrap(); + let expected_size = 4 + 8 + committee.len() * crypto::MAC_TAG_SIZE; + assert_eq!(encoded.len(), expected_size); + + let decoded: BlockAuthentication = bincode::deserialize(&encoded).unwrap(); + assert_eq!(decoded, *block.authentication()); + + let yaml = serde_yaml::to_string(block.authentication()).unwrap(); + let decoded_yaml: BlockAuthentication = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!(decoded_yaml, *block.authentication()); + } + + #[test] + fn flat_mac_vector_rejects_partial_tags() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + + let mut encoded = bincode::serialize(block.authentication()).unwrap(); + let invalid_payload_len = committee.len() * crypto::MAC_TAG_SIZE - 1; + encoded[4..12].copy_from_slice(&(invalid_payload_len as u64).to_le_bytes()); + encoded.truncate(12 + invalid_payload_len); + + assert!(bincode::deserialize::(&encoded).is_err()); + } + #[test] fn rejects_incomplete_mac_vector_and_wrong_authentication_scheme() { let committee = Committee::new_for_benchmarks(4); From 376ca41e8c31dd82dd36e1bbbb1c4406ad7e9b6d Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:53:55 +0200 Subject: [PATCH 03/62] Send recipient MACs on relay paths --- README.md | 11 ++- crates/starfish-core/src/broadcaster.rs | 98 ++++++++++++++++++++++- crates/starfish-core/src/types.rs | 100 +++++++++++++++++++++--- crates/starfish-core/src/validator.rs | 1 + 4 files changed, 191 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e4cce090..68f2f94e 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,13 @@ schemes: For all three variants, `BlockReference.digest` is the BLAKE3 hash of the canonical block content only. The authentication proof is a separate header -field and does not change the block reference. A `starfish-mac` block carries -exactly one tag for every committee member; each receiver verifies only its -own tag. Benchmark genesis deterministically generates the pairwise MAC keys, -ML-DSA seeds, and public keys in the node configuration. +field and does not change the block reference. A `starfish-mac` author sends +the full vector, with exactly one tag for every committee member, to its direct +recipients. A direct recipient retains that vector and, when relaying a header +or answering a missing-parent request, sends only the destination's tag. A +tag-only copy cannot be relayed a second time. Benchmark genesis +deterministically generates the pairwise MAC keys, ML-DSA seeds, and public +keys in the node configuration. This is research/benchmark code. The RustCrypto `ml-dsa` implementation used here states that it has not been independently audited and should not be diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index d7770fc3..b5f7a200 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -26,11 +26,43 @@ use crate::{ runtime::{Handle, sleep}, syncer::CommitObserver, types::{ - AuthorityIndex, AuthoritySet, BlockReference, RoundNumber, VerifiedBlock, - format_authority_index, + AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockReference, RoundNumber, + VerifiedBlock, format_authority_index, }, }; +/// Prepare blocks sent through relay and missing-parent response paths for a +/// specific peer. In the MAC experiment, a direct recipient retains the +/// complete vector and selects only the destination's tag for these paths. A +/// tag-only copy cannot be relayed a second time and is therefore omitted. +fn prepare_relay_blocks_for_peer( + authentication_scheme: BlockAuthenticationScheme, + recipient: AuthorityIndex, + blocks: Vec>, +) -> Vec> { + if authentication_scheme != BlockAuthenticationScheme::MacVector { + return blocks; + } + + blocks + .into_iter() + .filter_map(|block| { + block + .with_recipient_mac(recipient) + .map(Data::new) + .or_else(|| { + tracing::debug!( + "Cannot relay MAC-authenticated block {} to authority {}: \ + complete MAC vector is unavailable", + block.reference(), + recipient, + ); + None + }) + }) + .collect() +} + fn peer_can_serve_missing_data( consensus_protocol: ConsensusProtocol, holders: &StakeAggregator, @@ -430,6 +462,11 @@ where .inner .dag_state .get_transmission_parts(&refs_to_send, &refs_to_send); + let headers = prepare_relay_blocks_for_peer( + self.inner.dag_state.block_authentication_scheme, + peer_id, + headers, + ); { let mut sent = self.sent_to_peer.write(); for block in headers.iter() { @@ -1308,6 +1345,7 @@ where fn materialize_push_batch( inner: &Arc>, + to_whom_authority_index: AuthorityIndex, plan: PushBatchParts, ) -> BlockBatch where @@ -1337,6 +1375,11 @@ where let (headers, shards) = inner .dag_state .get_transmission_parts(&plan.other_refs, &plan.shard_refs); + let headers = prepare_relay_blocks_for_peer( + inner.dag_state.block_authentication_scheme, + to_whom_authority_index, + headers, + ); BlockBatch { source: DataSource::BlockBundleStreaming, full_blocks: plan.own_blocks, @@ -1417,7 +1460,7 @@ where // Drop own blocks from the plan — already shipped in the fast batch. plan.own_blocks = Vec::new(); - let slow_batch = materialize_push_batch(&inner, plan); + let slow_batch = materialize_push_batch(&inner, to_whom_authority_index, plan); if slow_batch.is_empty() { return Some(()); } @@ -1526,7 +1569,11 @@ impl BlockFetcherWorker { #[cfg(test)] mod tests { use super::*; - use crate::committee::Committee; + use crate::{ + committee::Committee, + crypto::{SignatureBytes, mac_keyrings_for_test}, + types::{BaseTransaction, BlockAuthentication}, + }; fn holder_set(authorities: &[AuthorityIndex]) -> StakeAggregator { let committee = Committee::new_test(vec![1, 1, 1, 1]); @@ -1574,4 +1621,47 @@ mod tests { assert_eq!(ramp_up_chain_bomb_release_probability(180.0), 1.0); assert_eq!(ramp_up_chain_bomb_release_probability(240.0), 1.0); } + + #[test] + fn relay_preparation_selects_recipient_tag_and_stops_after_one_hop() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = mac_keyrings_for_test(committee.len()); + let mut block = VerifiedBlock::new( + 0, + 1, + Vec::new(), + Vec::new(), + 0, + SignatureBytes::default(), + Vec::::new(), + None, + None, + None, + None, + ); + let tags: Vec<_> = keyrings[0] + .iter() + .enumerate() + .map(|(recipient, key)| { + key.compute_tag(0, recipient as AuthorityIndex, &block.digest()) + }) + .collect(); + let expected = tags[2]; + block.header.authentication = BlockAuthentication::MacVector(tags); + + let relayed = prepare_relay_blocks_for_peer( + BlockAuthenticationScheme::MacVector, + 2, + vec![Data::new(block)], + ); + assert_eq!(relayed.len(), 1); + assert!(matches!( + relayed[0].authentication(), + BlockAuthentication::MacTag(tag) if *tag == expected + )); + + let second_hop = + prepare_relay_blocks_for_peer(BlockAuthenticationScheme::MacVector, 3, relayed); + assert!(second_hop.is_empty()); + } } diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 53177a24..48b8a039 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -258,7 +258,10 @@ pub enum BlockAuthentication { /// Only valid for locally constructed genesis blocks. None, Ed25519(SignatureBytes), + /// Complete author-generated authenticator retained by direct recipients. MacVector(#[serde(with = "flat_mac_vector")] Vec), + /// Recipient-specific authenticator selected from a full vector by a relay. + MacTag(MacTag), MlDsa44(MlDsa44SignatureBytes), } @@ -1297,6 +1300,21 @@ impl VerifiedBlock { } } + /// Clone a block for relaying to `recipient`, replacing its complete MAC + /// vector with only that recipient's tag. A block that was itself received + /// with a single tag cannot be relayed again. + pub fn with_recipient_mac(&self, recipient: AuthorityIndex) -> Option { + let BlockAuthentication::MacVector(tags) = &self.header.authentication else { + return None; + }; + let tag = *tags.get(recipient as usize)?; + + let mut block = self.clone(); + block.header.authentication = BlockAuthentication::MacTag(tag); + block.header.serialized = None; + Some(block) + } + // --- Decomposition --- /// Extract the header, consuming self. @@ -1507,13 +1525,23 @@ impl VerifiedBlock { bail!("Block Ed25519 verification has failed: {error:?}"); } } - (BlockAuthenticationScheme::MacVector, BlockAuthentication::MacVector(tags)) => { - ensure!( - tags.len() == committee.len(), - "MAC vector length {} does not match committee size {}", - tags.len(), - committee.len(), - ); + (BlockAuthenticationScheme::MacVector, authentication) => { + let tag = match authentication { + BlockAuthentication::MacVector(tags) => { + ensure!( + tags.len() == committee.len(), + "MAC vector length {} does not match committee size {}", + tags.len(), + committee.len(), + ); + tags.get(own_id) + .ok_or_else(|| eyre::eyre!("Own authority index is out of bounds"))? + } + BlockAuthentication::MacTag(tag) => tag, + actual => { + bail!("Expected MacVector block authentication, received {actual:?}") + } + }; ensure!( own_id < committee.len(), "Own authority index is out of bounds" @@ -1529,10 +1557,7 @@ impl VerifiedBlock { bail!("Unknown block author {}", self.authority()) }; let expected = key.compute_tag(self.authority(), own_id as AuthorityIndex, &digest); - ensure!( - tags[own_id] == expected, - "Block MAC verification has failed" - ); + ensure!(*tag == expected, "Block MAC verification has failed"); } (BlockAuthenticationScheme::MlDsa44, BlockAuthentication::MlDsa44(signature)) => { let Some(public_key) = committee.get_ml_dsa_44_public_key(self.authority()) else { @@ -2493,6 +2518,59 @@ mod tests { assert_eq!(decoded_yaml, *block.authentication()); } + #[test] + fn relay_selects_only_the_destination_mac() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + let BlockAuthentication::MacVector(full_vector) = block.authentication() else { + panic!("expected full MAC vector") + }; + + let mut relayed = block.with_recipient_mac(2).unwrap(); + let BlockAuthentication::MacTag(tag) = relayed.authentication() else { + panic!("expected recipient MAC tag") + }; + assert_eq!(*tag, full_vector[2]); + assert_eq!(relayed.reference(), block.reference()); + assert_eq!( + bincode::serialize(relayed.authentication()).unwrap().len(), + 4 + 8 + crypto::MAC_TAG_SIZE, + ); + + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + relayed + .verify_with_authentication( + &committee, + 2, + 1, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[2], + ) + .unwrap(); + + let mut wrong_recipient = block.with_recipient_mac(2).unwrap(); + assert!( + wrong_recipient + .verify_with_authentication( + &committee, + 1, + 2, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[1], + ) + .is_err() + ); + assert!(relayed.with_recipient_mac(3).is_none()); + } + #[test] fn flat_mac_vector_rejects_partial_tags() { let committee = Committee::new_for_benchmarks(4); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 420acca5..5a71a479 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -443,6 +443,7 @@ mod smoke_tests { #[test_case("mysticeti", 100)] #[test_case("cordial-miners", 140)] #[test_case("starfish", 160)] + #[test_case("starfish-mac", 740)] #[test_case("starfish-speed", 180)] #[test_case("starfish-bls", 200)] #[test_case("sailfish++", 220)] From a1391fc38c10aa4c0410cd0eb739926d07a2b94e Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:30:40 +0200 Subject: [PATCH 04/62] Upgrade stored recipient MACs to full vectors --- README.md | 8 +- crates/starfish-core/src/block_manager.rs | 183 +++++++++++++++++++++- crates/starfish-core/src/dag_state.rs | 132 +++++++++++++++- crates/starfish-core/src/net_sync.rs | 131 ++++++++++++---- crates/starfish-core/src/types.rs | 68 ++++++++ 5 files changed, 479 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 68f2f94e..eea8e2bb 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,11 @@ field and does not change the block reference. A `starfish-mac` author sends the full vector, with exactly one tag for every committee member, to its direct recipients. A direct recipient retains that vector and, when relaying a header or answering a missing-parent request, sends only the destination's tag. A -tag-only copy cannot be relayed a second time. Benchmark genesis -deterministically generates the pairwise MAC keys, ML-DSA seeds, and public -keys in the node configuration. +tag-only copy cannot be relayed a second time. If the same node later receives +the author's full-vector copy, it upgrades the stored authentication without +adding a second DAG vertex and can then relay recipient-specific tags. +Benchmark genesis deterministically generates the pairwise MAC keys, ML-DSA +seeds, and public keys in the node configuration. This is research/benchmark code. The RustCrypto `ml-dsa` implementation used here states that it has not been independently audited and should not be diff --git a/crates/starfish-core/src/block_manager.rs b/crates/starfish-core/src/block_manager.rs index f05e8d5f..4b34b864 100644 --- a/crates/starfish-core/src/block_manager.rs +++ b/crates/starfish-core/src/block_manager.rs @@ -56,6 +56,12 @@ impl BlockManager { let mut updated_existing_with_transactions: Vec> = vec![]; // Blocks to insert into the DAG in a single batched write lock. let mut blocks_to_insert: Vec> = vec![]; + // References first discovered in this batch are not visible through + // DagState until the final batched insert. Keep their positions so a + // richer duplicate later in the same batch can upgrade that pending + // insertion instead of being mistaken for an already stored block. + let mut new_blocks_in_batch: AHashMap = AHashMap::new(); + let mut updated_blocks_in_batch: AHashMap = AHashMap::new(); // missing references that we don't currently have let mut missing_references = AHashSet::new(); let mut block_exists_cache: AHashMap = AHashMap::new(); @@ -63,8 +69,33 @@ impl BlockManager { let block_reference = block.reference(); if let Some(existing_pending_block) = self.blocks_pending.get_mut(block_reference) { - if block.transactions().is_some() { - *existing_pending_block = block; + if let Some(mut merged) = existing_pending_block.merge_same_block(&block) { + merged.preserialize(); + *existing_pending_block = Data::new(merged); + } + continue; + } + + if let Some((insert_index, updated_index)) = + updated_blocks_in_batch.get(block_reference).copied() + { + if let Some(mut merged) = blocks_to_insert[insert_index].merge_same_block(&block) { + merged.preserialize(); + let merged = Data::new(merged); + blocks_to_insert[insert_index] = merged.clone(); + updated_existing_with_transactions[updated_index] = merged; + } + continue; + } + + if let Some((insert_index, processed_index)) = + new_blocks_in_batch.get(block_reference).copied() + { + if let Some(mut merged) = blocks_to_insert[insert_index].merge_same_block(&block) { + merged.preserialize(); + let merged = Data::new(merged); + blocks_to_insert[insert_index] = merged.clone(); + newly_processed[processed_index] = merged; } continue; } @@ -76,8 +107,24 @@ impl BlockManager { // Block already in store — check if this version brings new transaction data if self.dag_state.contains_new_transactions(&block) { tracing::debug!("Block has new transactions: {:?}", block_reference); + let stored_reference = *block_reference; + let mut merged = self + .dag_state + .get_storage_block(stored_reference) + .and_then(|existing| existing.merge_same_block(&block)); + let block = if let Some(ref mut merged) = merged { + merged.preserialize(); + Data::new(merged.clone()) + } else { + block + }; + let insert_index = blocks_to_insert.len(); blocks_to_insert.push(block.clone()); + let updated_index = updated_existing_with_transactions.len(); updated_existing_with_transactions.push(block); + updated_blocks_in_batch.insert(stored_reference, (insert_index, updated_index)); + } else { + self.dag_state.upgrade_mac_authentication(&block); } continue; } @@ -122,9 +169,12 @@ impl BlockManager { let block_reference = *block_reference; // Defer DAG insertion — will be done in batch after the loop. + let insert_index = blocks_to_insert.len(); blocks_to_insert.push(block.clone()); block_exists_cache.insert(block_reference, true); + let processed_index = newly_processed.len(); newly_processed.push(block); + new_blocks_in_batch.insert(block_reference, (insert_index, processed_index)); // Now unlock any pending blocks, and process them if ready. if let Some(waiting_references) = @@ -194,3 +244,132 @@ impl BlockManager { /// evicting unresolved chains from the block manager. pub fn cleanup(&mut self, _threshold_round: RoundNumber) {} } + +#[cfg(test)] +mod tests { + use prometheus::Registry; + use tempfile::TempDir; + + use super::*; + use crate::{ + config::{DisseminationMode, StorageBackend}, + crypto, + dag_state::ConsensusProtocol, + metrics::Metrics, + types::{AuthorityIndex, BlockAuthorizer}, + }; + + fn open_mac_dag_state(committee: Arc, path: &std::path::Path) -> DagState { + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-mac"), + None, + ); + DagState::open( + 0, + path, + metrics, + committee, + "honest".to_string(), + "starfish-mac".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ) + .dag_state + } + + fn make_mac_block( + keyrings: &[Vec], + authority: AuthorityIndex, + round: RoundNumber, + parents: Vec, + ) -> VerifiedBlock { + let mut block = VerifiedBlock::new_with_authorizer_and_unprovable( + authority, + round, + parents, + None, + Vec::new(), + 0, + &BlockAuthorizer::MacVector(&keyrings[authority as usize]), + None, + None, + Vec::new(), + Vec::new(), + None, + ConsensusProtocol::Starfish, + None, + None, + None, + None, + None, + None, + None, + ); + block.preserialize(); + block + } + + #[test] + fn block_manager_upgrades_stored_batched_and_pending_mac_copies() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let temp_dir = TempDir::new().unwrap(); + let dag_state = open_mac_dag_state(committee.clone(), temp_dir.path()); + let mut manager = BlockManager::new(dag_state.clone(), &committee); + let genesis: Vec<_> = committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(); + + // A stored tag-only copy is upgraded when the author's full vector + // arrives later, without reporting another newly processed block. + let full = make_mac_block(&keyrings, 1, 1, genesis.clone()); + let reference = *full.reference(); + let mut tagged = full.with_recipient_mac(0).unwrap(); + tagged.preserialize(); + assert_eq!( + manager + .add_blocks(vec![Data::new(tagged)], DataSource::BlockBundleStreaming,) + .0 + .len(), + 1 + ); + assert!( + manager + .add_blocks(vec![Data::new(full)], DataSource::BlockBundleStreaming,) + .0 + .is_empty() + ); + assert!( + dag_state + .get_storage_block(reference) + .unwrap() + .has_full_mac_vector() + ); + + // The same upgrade also works when both copies share one receive + // batch and when the block is waiting on a missing parent. + let parent = make_mac_block(&keyrings, 2, 1, genesis); + let child = make_mac_block(&keyrings, 2, 2, vec![*parent.reference()]); + let child_reference = *child.reference(); + let mut tagged_child = child.with_recipient_mac(0).unwrap(); + tagged_child.preserialize(); + manager.add_blocks( + vec![Data::new(tagged_child), Data::new(child)], + DataSource::BlockBundleStreaming, + ); + assert_eq!(manager.pending_blocks_count(), 1); + manager.add_blocks(vec![Data::new(parent)], DataSource::BlockBundleStreaming); + assert_eq!(manager.pending_blocks_count(), 0); + assert!( + dag_state + .get_storage_block(child_reference) + .unwrap() + .has_full_mac_vector() + ); + } +} diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index be428fb0..cb9e44a3 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -32,8 +32,8 @@ use crate::{ store::Store, threshold_clock::ThresholdClockAggregator, types::{ - AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - BlsAggregateCertificate, ProvableShard, RoundNumber, SailfishNoVoteCert, + AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, + BlockReference, BlsAggregateCertificate, ProvableShard, RoundNumber, SailfishNoVoteCert, SailfishTimeoutCert, TransactionData, VerifiedBlock, }, }; @@ -1177,6 +1177,66 @@ impl DagState { self.dag_state_inner.read().get_storage_block(reference) } + /// Upgrade an already stored recipient-only MAC copy with a later verified + /// full-vector copy. This intentionally updates only the persisted header + /// and the matching in-memory value: the block is not re-added to the DAG, + /// so threshold clocks, votes, consensus notifications, and acceptance + /// metrics are left untouched. + pub(crate) fn upgrade_mac_authentication(&self, incoming: &VerifiedBlock) -> bool { + if !incoming.has_full_mac_vector() { + return false; + } + + let reference = *incoming.reference(); + let Some(existing) = self.get_storage_block(reference) else { + return false; + }; + if !matches!(existing.authentication(), BlockAuthentication::MacTag(_)) { + return false; + } + let Some(mut upgraded) = existing.merge_same_block(incoming) else { + return false; + }; + upgraded.preserialize(); + + let store_start = std::time::Instant::now(); + self.store + .store_header_bytes( + upgraded.reference(), + upgraded + .serialized_header_bytes() + .expect("upgraded header should be preserialized"), + ) + .expect("Failed to store upgraded MAC-vector header"); + self.metrics + .store_block_latency_us + .inc_by(store_start.elapsed().as_micros() as u64); + self.metrics.store_block_count.inc(); + + // Preserve any transaction data that may have arrived concurrently + // with the authentication upgrade. + let mut inner = self.dag_state_inner.write(); + let authority = reference.authority as usize; + let Some(blocks_at_round) = inner.index[authority].get_mut(&reference.round) else { + // The block was evicted; the persistent header update above is the + // authoritative copy and it should remain evicted from memory. + return true; + }; + let Some(current) = blocks_at_round.get_mut(&reference.digest) else { + return true; + }; + if matches!(current.authentication(), BlockAuthentication::MacTag(_)) { + let mut memory_upgrade = current + .merge_same_block(incoming) + .expect("tag-only copy should accept a full-vector upgrade"); + memory_upgrade.preserialize(); + *current = Data::new(memory_upgrade); + *inner.round_version.entry(reference.round).or_insert(0) += 1; + } + + true + } + /// Look up the `transactions_commitment` for a block in the DAG. pub fn get_transactions_commitment( &self, @@ -3445,15 +3505,16 @@ mod tests { committee::Committee, config::{DisseminationMode, StorageBackend}, crypto::{ - BLS_SIGNATURE_SIZE, BlockDigest, BlsSignatureBytes, SignatureBytes, + self, BLS_SIGNATURE_SIZE, BlockDigest, BlsSignatureBytes, SignatureBytes, TransactionsCommitment, }, data::Data, metrics::Metrics, types::{ - AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, - BlockReference, BlsAggregateCertificate, ProvableShard, RoundNumber, SailfishFields, - SailfishNoVoteCert, Transaction, VerifiedBlock, + AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthentication, + BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, + ProvableShard, RoundNumber, SailfishFields, SailfishNoVoteCert, Transaction, + VerifiedBlock, }, }; @@ -3710,6 +3771,65 @@ mod tests { ); } + #[test] + fn full_mac_vector_upgrades_tag_only_block_in_memory_and_storage() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let mut full = VerifiedBlock::new_with_authorizer_and_unprovable( + 1, + 1, + committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(), + None, + Vec::new(), + 0, + &BlockAuthorizer::MacVector(&keyrings[1]), + None, + None, + Vec::new(), + Vec::new(), + None, + ConsensusProtocol::Starfish, + None, + None, + None, + None, + None, + None, + None, + ); + full.preserialize(); + let reference = *full.reference(); + + let mut tagged = full.with_recipient_mac(0).unwrap(); + tagged.preserialize(); + let dag_state = open_test_dag_state_for("starfish-mac", 0); + dag_state.insert_general_block(Data::new(tagged), DataSource::BlockBundleStreaming); + + assert!(matches!( + dag_state + .get_storage_block(reference) + .unwrap() + .authentication(), + BlockAuthentication::MacTag(_) + )); + assert!(matches!( + dag_state.get_blocks_by_round_cached(1)[0].authentication(), + BlockAuthentication::MacTag(_) + )); + assert!(dag_state.upgrade_mac_authentication(&full)); + + let upgraded = dag_state.get_storage_block(reference).unwrap(); + assert!(upgraded.has_full_mac_vector()); + assert!(upgraded.with_recipient_mac(2).is_some()); + assert!(dag_state.get_blocks_by_round_cached(1)[0].has_full_mac_vector()); + let persisted = dag_state.store.get_block(&reference).unwrap().unwrap(); + assert!(persisted.has_full_mac_vector()); + assert!(!dag_state.upgrade_mac_authentication(&full)); + } + #[test] fn batch_vertex_certification_waits_for_parent_closure() { let dag_state = open_test_dag_state_for("sailfish-pp", 0); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 3b95bf62..2ae7e943 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -161,6 +161,7 @@ fn eligible_missing_parent_refs( struct FilterForBlocks { digests: parking_lot::RwLock>, + full_mac_vectors: parking_lot::RwLock>, queue: parking_lot::RwLock>, } @@ -168,6 +169,7 @@ impl FilterForBlocks { fn new() -> Self { Self { digests: parking_lot::RwLock::new(AHashSet::new()), + full_mac_vectors: parking_lot::RwLock::new(AHashSet::new()), queue: parking_lot::RwLock::new(VecDeque::new()), } } @@ -177,58 +179,84 @@ impl FilterForBlocks { digests.iter().map(|d| set.contains(d)).collect() } - fn insert_batch(&self, new_digests: &[BlockDigest]) { + fn contains_full_mac_batch(&self, digests: &[BlockDigest]) -> Vec { + let set = self.full_mac_vectors.read(); + digests.iter().map(|d| set.contains(d)).collect() + } + + fn insert_batch(&self, blocks: &[(BlockDigest, bool)]) { let mut digests = self.digests.write(); + let mut full_mac_vectors = self.full_mac_vectors.write(); let mut queue = self.queue.write(); - for digest in new_digests { + for (digest, has_full_mac_vector) in blocks { if digests.insert(*digest) { queue.push_back(*digest); } + if *has_full_mac_vector { + full_mac_vectors.insert(*digest); + } } while queue.len() > MAX_FILTER_SIZE { if let Some(removed) = queue.pop_front() { digests.remove(&removed); + full_mac_vectors.remove(&removed); } } } - /// Inserts all digests and returns `true` for each that was genuinely new - /// (not already in the filter and not duplicated earlier in the batch). - fn insert_and_report_new(&self, digests: &[BlockDigest]) -> Vec { + /// Inserts all verified copies and returns `true` for each copy that adds + /// either a new block reference or the first full MAC vector for a + /// previously recipient-tag-only reference. + fn insert_and_report_useful(&self, blocks: &[(BlockDigest, bool)]) -> Vec { let mut set = self.digests.write(); + let mut full_mac_vectors = self.full_mac_vectors.write(); let mut queue = self.queue.write(); - let is_new: Vec = digests + let is_useful: Vec = blocks .iter() - .map(|d| { - if set.insert(*d) { - queue.push_back(*d); - true - } else { - false + .map(|(digest, has_full_mac_vector)| { + let is_new = set.insert(*digest); + if is_new { + queue.push_back(*digest); } + let is_mac_upgrade = *has_full_mac_vector && full_mac_vectors.insert(*digest); + is_new || is_mac_upgrade }) .collect(); while queue.len() > MAX_FILTER_SIZE { if let Some(removed) = queue.pop_front() { set.remove(&removed); + full_mac_vectors.remove(&removed); } } - is_new + is_useful } - /// For each header digest, returns `true` if the digest has not been seen - /// before (neither in the filter nor earlier in this batch). - fn needed_headers(&self, batch: &[BlockDigest]) -> Vec { + /// For each header, returns `true` if it is either unseen or upgrades a + /// previously seen recipient-only MAC to a full vector. + fn needed_headers(&self, batch: &[(BlockDigest, bool)]) -> Vec { let digests = self.digests.read(); - let mut seen_in_batch = AHashSet::with_capacity(batch.len()); + let full_mac_vectors = self.full_mac_vectors.read(); + let mut seen_in_batch = AHashMap::with_capacity(batch.len()); batch .iter() - .map(|digest| !digests.contains(digest) && seen_in_batch.insert(*digest)) + .map(|(digest, has_full_mac_vector)| { + let was_seen = digests.contains(digest) || seen_in_batch.contains_key(digest); + let had_full_mac_vector = seen_in_batch + .get(digest) + .copied() + .unwrap_or_else(|| full_mac_vectors.contains(digest)); + let is_needed = !was_seen || (*has_full_mac_vector && !had_full_mac_vector); + seen_in_batch + .entry(*digest) + .and_modify(|full| *full |= *has_full_mac_vector) + .or_insert(*has_full_mac_vector); + is_needed + }) .collect() } } @@ -424,8 +452,11 @@ fn spawn_header_worker( let mut encoder = ReedSolomonEncoder::new(2, 4, 2).expect("Encoder should be created"); while let Some((blocks, source)) = rx.recv().await { let connection_knowledge = inner.cordial_knowledge.connection_knowledge(peer_id); - let incoming_digests: Vec<_> = blocks.iter().map(|block| block.digest()).collect(); - let needed_before_verify = filter_for_blocks.needed_headers(&incoming_digests); + let incoming_headers: Vec<_> = blocks + .iter() + .map(|block| (block.digest(), block.has_full_mac_vector())) + .collect(); + let needed_before_verify = filter_for_blocks.needed_headers(&incoming_headers); let mut verified_blocks: Vec = Vec::new(); for (data_block, is_needed) in blocks.into_iter().zip(needed_before_verify) { @@ -466,11 +497,14 @@ fn spawn_header_worker( ck.mark_headers_useful_from_peer(&refs); } - let digests: Vec<_> = verified_blocks.iter().map(|b| b.digest()).collect(); - let is_new = filter_for_blocks.insert_and_report_new(&digests); + let filter_entries: Vec<_> = verified_blocks + .iter() + .map(|block| (block.digest(), block.has_full_mac_vector())) + .collect(); + let is_useful = filter_for_blocks.insert_and_report_useful(&filter_entries); let mut new_data_blocks = Vec::new(); - for (storage_block, is_new) in verified_blocks.into_iter().zip(is_new) { - if is_new { + for (storage_block, is_useful) in verified_blocks.into_iter().zip(is_useful) { + if is_useful { let mut storage_block = storage_block; storage_block.preserialize(); debug_assert!( @@ -962,16 +996,18 @@ impl ConnectionHandler)> = Vec::new(); - for ((data_block, _digest), (bk, sf)) in blocks - .into_iter() - .zip(incoming_digests) - .zip(block_known.into_iter().zip(shard_full)) - { - if bk && sf { + for (index, data_block) in blocks.into_iter().enumerate() { + let bk = block_known[index]; + let sf = shard_full[index]; + let incoming_has_full_mac = data_block.has_full_mac_vector(); + if bk && sf && (!incoming_has_full_mac || full_mac_known[index]) { self.metrics.filtered_blocks_total.inc(); continue; } @@ -1012,8 +1048,16 @@ impl ConnectionHandler = verified.iter().map(|(b, _)| b.digest()).collect(); - self.filter_for_blocks.insert_batch(&verified_digests); + let verified_filter_entries: Vec<_> = verified + .iter() + .map(|(block, _)| (block.digest(), block.has_full_mac_vector())) + .collect(); + let verified_digests: Vec<_> = verified_filter_entries + .iter() + .map(|(digest, _)| *digest) + .collect(); + self.filter_for_blocks + .insert_batch(&verified_filter_entries); self.filter_for_shards.mark_full_batch(&verified_digests); // --- preserialize + collect --- @@ -2388,6 +2432,29 @@ mod tests { wait.await; } + #[test] + fn block_filter_allows_exactly_one_tag_to_full_mac_upgrade() { + let filter = FilterForBlocks::new(); + let digest = BlockReference::new_test(1, 7).digest; + + assert_eq!( + filter.needed_headers(&[(digest, false), (digest, true), (digest, true)]), + vec![true, true, false] + ); + assert_eq!( + filter.insert_and_report_useful(&[(digest, false)]), + vec![true] + ); + assert_eq!(filter.needed_headers(&[(digest, false)]), vec![false]); + assert_eq!(filter.needed_headers(&[(digest, true)]), vec![true]); + assert_eq!( + filter.insert_and_report_useful(&[(digest, true), (digest, true)]), + vec![true, false] + ); + assert_eq!(filter.needed_headers(&[(digest, true)]), vec![false]); + assert_eq!(filter.contains_full_mac_batch(&[digest]), vec![true]); + } + #[test] fn acknowledgments_imply_peer_knows_shard_data() { let ack_ref = BlockReference::new_test(2, 3); diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 48b8a039..8f8830e1 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -1292,6 +1292,48 @@ impl VerifiedBlock { self.transaction_data.is_some() } + /// Returns whether this copy retains the author's complete MAC vector and + /// can therefore be specialized for another recipient. + pub fn has_full_mac_vector(&self) -> bool { + matches!( + &self.header.authentication, + BlockAuthentication::MacVector(_) + ) + } + + /// Merge two verified copies of the same content-addressed block, keeping + /// the richest independently transported components from either copy: + /// transaction data and the author's complete MAC vector. + /// + /// Returns `None` when the references differ or the merge adds nothing. + pub fn merge_same_block(&self, incoming: &Self) -> Option { + if self.reference() != incoming.reference() { + return None; + } + + let mut merged = self.clone(); + let mut changed = false; + + if merged.transaction_data.is_none() && incoming.transaction_data.is_some() { + merged.transaction_data = incoming.transaction_data.clone(); + changed = true; + } + + if matches!( + &merged.header.authentication, + BlockAuthentication::MacTag(_) + ) && matches!( + &incoming.header.authentication, + BlockAuthentication::MacVector(_) + ) { + merged.header.authentication = incoming.header.authentication.clone(); + merged.header.serialized = None; + changed = true; + } + + changed.then_some(merged) + } + /// Create a lightweight copy with only the header (no transaction data). pub fn as_header_only(&self) -> Self { Self { @@ -2571,6 +2613,32 @@ mod tests { assert!(relayed.with_recipient_mac(3).is_none()); } + #[test] + fn same_block_merge_keeps_full_mac_and_transaction_data_in_either_order() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let full = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + let mut tagged_with_transactions = full.with_recipient_mac(1).unwrap(); + tagged_with_transactions.transaction_data = + Some(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![1, 2, 3]), + )])); + + let tag_then_full = tagged_with_transactions.merge_same_block(&full).unwrap(); + assert!(tag_then_full.has_full_mac_vector()); + assert!(tag_then_full.has_transaction_data()); + + let full_then_tag = full + .as_header_only() + .merge_same_block(&tagged_with_transactions) + .unwrap(); + assert!(full_then_tag.has_full_mac_vector()); + assert!(full_then_tag.has_transaction_data()); + } + #[test] fn flat_mac_vector_rejects_partial_tags() { let committee = Committee::new_for_benchmarks(4); From 4a89caea26c0eca2c99bbc5db325eba365645c2d Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:42:09 +0200 Subject: [PATCH 05/62] Restrict full MAC vectors to author streams --- README.md | 13 ++- crates/starfish-core/src/net_sync.rs | 168 ++++++++++++++++++++++++++- crates/starfish-core/src/types.rs | 97 +++++++++++++++- 3 files changed, 269 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index eea8e2bb..fb89d150 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,14 @@ field and does not change the block reference. A `starfish-mac` author sends the full vector, with exactly one tag for every committee member, to its direct recipients. A direct recipient retains that vector and, when relaying a header or answering a missing-parent request, sends only the destination's tag. A -tag-only copy cannot be relayed a second time. If the same node later receives -the author's full-vector copy, it upgrades the stored authentication without -adding a second DAG vertex and can then relay recipient-specific tags. -Benchmark genesis deterministically generates the pairwise MAC keys, ML-DSA -seeds, and public keys in the node configuration. +tag-only copy cannot be relayed a second time. Receivers accept a full vector +only through proactive block streaming directly from the block's claimed +author; relay and synchronization traffic must contain exactly one recipient +tag. If the same node later receives the author's directly streamed +full-vector copy, it upgrades the stored authentication without adding a +second DAG vertex and can then relay recipient-specific tags. Benchmark +genesis deterministically generates the pairwise MAC keys, ML-DSA seeds, and +public keys in the node configuration. This is research/benchmark code. The RustCrypto `ml-dsa` implementation used here states that it has not been independently audited and should not be diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 2ae7e943..5890fedf 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -46,8 +46,9 @@ use crate::{ shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ - AuthorityIndex, AuthoritySet, BlockDigest, BlockReference, PartialSig, PartialSigKind, - ProvableShard, RoundNumber, VerifiedBlock, format_authority_index, + AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, + BlockReference, PartialSig, PartialSigKind, ProvableShard, RoundNumber, VerifiedBlock, + format_authority_index, }, }; @@ -55,6 +56,48 @@ const MAX_FILTER_SIZE: usize = 100_000; const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; +/// Enforce the MAC experiment's transport contract before cryptographic +/// verification: +/// +/// - a full vector is accepted only on proactive block streaming directly +/// from the block's claimed author; +/// - every relay and synchronization path must carry one recipient tag; +/// - a direct author stream must carry the full vector, so recipients retain +/// the material needed for one-hop relay. +fn verify_mac_transport( + block: &VerifiedBlock, + authentication_scheme: BlockAuthenticationScheme, + peer_id: AuthorityIndex, + source: DataSource, +) -> eyre::Result<()> { + if authentication_scheme != BlockAuthenticationScheme::MacVector { + return Ok(()); + } + + let direct_author_stream = peer_id == block.authority() + && matches!( + source, + DataSource::BlockBundleStreaming | DataSource::BlockBundleStreamingHeader + ); + + match block.authentication() { + BlockAuthentication::MacVector(_) if direct_author_stream => Ok(()), + BlockAuthentication::MacVector(_) => eyre::bail!( + "Full MAC vector for block {} must arrive via direct author block streaming; \ + received from authority {} with source {}", + block.reference(), + peer_id, + source, + ), + BlockAuthentication::MacTag(_) if !direct_author_stream => Ok(()), + BlockAuthentication::MacTag(_) => eyre::bail!( + "Direct author block stream for block {} must carry the full MAC vector", + block.reference(), + ), + _ => Ok(()), + } +} + async fn send_network_message_reliably( sender: &mpsc::Sender, message: NetworkMessage, @@ -466,6 +509,20 @@ fn spawn_header_worker( } let mut block: VerifiedBlock = (*data_block).clone(); tracing::debug!("Received {} from {}", block, peer); + if let Err(e) = verify_mac_transport( + &block, + inner.dag_state.block_authentication_scheme, + peer_id, + source, + ) { + tracing::warn!( + "Rejected incorrectly transported block {} from {}: {:?}", + block.reference(), + peer, + e + ); + break; + } match block.verify_with_authentication( &inner.committee, own_id as usize, @@ -1013,6 +1070,20 @@ impl ConnectionHandler, ) -> VerifiedBlock { - let authority = 0; + make_authenticated_starfish_block_for_author(committee, 0, authorizer) + } + + fn make_authenticated_starfish_block_for_author( + committee: &Committee, + authority: AuthorityIndex, + authorizer: &BlockAuthorizer<'_>, + ) -> VerifiedBlock { let round = 1; let transactions = Vec::new(); let mut encoder = Encoder::new(2, 4, 2).unwrap(); @@ -2613,6 +2620,94 @@ mod tests { assert!(relayed.with_recipient_mac(3).is_none()); } + #[test] + fn mac_verification_authenticates_the_claimed_block_author() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let mut correctly_authenticated = make_authenticated_starfish_block_for_author( + &committee, + 1, + &BlockAuthorizer::MacVector(&keyrings[1]), + ); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + correctly_authenticated + .verify_with_authentication( + &committee, + 2, + 1, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[2], + ) + .unwrap(); + + // The content claims authority 1, but authority 0's pairwise keys + // produced the vector. Recipient 2 must reject it when selecting the + // key associated with the claimed author. + let mut wrong_author_keys = make_authenticated_starfish_block_for_author( + &committee, + 1, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + assert!( + wrong_author_keys + .verify_with_authentication( + &committee, + 2, + 1, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[2], + ) + .is_err() + ); + } + + #[test] + fn mac_vector_verification_is_limited_to_the_receivers_own_tag() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = crypto::mac_keyrings_for_test(committee.len()); + let block = make_authenticated_starfish_block( + &committee, + &BlockAuthorizer::MacVector(&keyrings[0]), + ); + let mut tampered = block.clone(); + let BlockAuthentication::MacVector(tags) = &mut tampered.header.authentication else { + panic!("expected full MAC vector") + }; + tags[3] = MacTag::from_bytes([0; crypto::MAC_TAG_SIZE]); + + let mut receiver_one = tampered.clone(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + receiver_one + .verify_with_authentication( + &committee, + 1, + 0, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[1], + ) + .unwrap(); + + assert!( + tampered + .verify_with_authentication( + &committee, + 3, + 0, + &mut encoder, + ConsensusProtocol::Starfish, + BlockAuthenticationScheme::MacVector, + &keyrings[3], + ) + .is_err() + ); + } + #[test] fn same_block_merge_keeps_full_mac_and_transaction_data_in_either_order() { let committee = Committee::new_for_benchmarks(4); From 00304cf8b1d2d95cef19fc8bb21cd2dd0d021265 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:52:11 +0200 Subject: [PATCH 06/62] Add Starfish Speed authentication variants --- README.md | 35 ++++++++++---------- crates/orchestrator/README.md | 2 +- crates/orchestrator/src/benchmark.rs | 1 + crates/orchestrator/src/main.rs | 3 +- crates/starfish-core/src/dag_state.rs | 30 +++++++++++++++++ crates/starfish-core/src/types.rs | 47 +++++++++++++++------------ crates/starfish-core/src/validator.rs | 4 +++ crates/starfish/src/main.rs | 12 +++---- 8 files changed, 87 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index fb89d150..b0b655c8 100644 --- a/README.md +++ b/README.md @@ -61,27 +61,26 @@ offloaded from the critical path. ### Starfish block authentication experiments -Plain Starfish can be run with three interchangeable block-authentication -schemes: +Plain Starfish and Starfish Speed can each be run with three interchangeable +block-authentication schemes: -| CLI name | Block authentication | -|---|---| -| `starfish` | Ed25519 signature | -| `starfish-mac` | Full vector of pairwise keyed-BLAKE3 MAC tags | -| `starfish-ml-dsa-44` | ML-DSA-44 signature | +| Protocol | Ed25519 | MAC vector | ML-DSA-44 | +|---|---|---|---| +| Starfish | `starfish` | `starfish-mac` | `starfish-ml-dsa-44` | +| Starfish Speed | `starfish-speed` | `starfish-speed-mac` | `starfish-speed-ml-dsa-44` | -For all three variants, `BlockReference.digest` is the BLAKE3 hash of the +For all six variants, `BlockReference.digest` is the BLAKE3 hash of the canonical block content only. The authentication proof is a separate header -field and does not change the block reference. A `starfish-mac` author sends -the full vector, with exactly one tag for every committee member, to its direct -recipients. A direct recipient retains that vector and, when relaying a header -or answering a missing-parent request, sends only the destination's tag. A -tag-only copy cannot be relayed a second time. Receivers accept a full vector -only through proactive block streaming directly from the block's claimed -author; relay and synchronization traffic must contain exactly one recipient -tag. If the same node later receives the author's directly streamed -full-vector copy, it upgrades the stored authentication without adding a -second DAG vertex and can then relay recipient-specific tags. Benchmark +field and does not change the block reference. An author using either MAC +variant sends the full vector, with exactly one tag for every committee member, +to its direct recipients. A direct recipient retains that vector and, when +relaying a header or answering a missing-parent request, sends only the +destination's tag. A tag-only copy cannot be relayed a second time. Receivers +accept a full vector only through proactive block streaming directly from the +block's claimed author; relay and synchronization traffic must contain exactly +one recipient tag. If the same node later receives the author's directly +streamed full-vector copy, it upgrades the stored authentication without adding +a second DAG vertex and can then relay recipient-specific tags. Benchmark genesis deterministically generates the pairwise MAC keys, ML-DSA seeds, and public keys in the node configuration. diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index ba884de6..db889f1f 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -129,7 +129,7 @@ each load generator submits a fixed load of 100 tx/s or more precisely 10 tx every 100ms. Performance measurements are collected by regularly scraping the Prometheus metrics exposed by the load generators. -Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `sparse-starfish-speed`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. +Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `sparse-starfish-speed`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index 6965d4a3..dd5cfedd 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -56,6 +56,7 @@ pub struct BenchmarkParametersGeneric { pub use_internal_ip_address: bool, // Consensus protocol to deploy // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-speed | + // starfish-speed-mac | starfish-speed-ml-dsa-44 | // sparse-starfish-speed | starfish-bls | // mysticeti | mysticeti-bls | cordial-miners | bluestreak | sailfish-pp) pub consensus_protocol: String, diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 545c63fb..c42b062c 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -136,7 +136,8 @@ pub enum Operation { /// Protocols to benchmark in order. Available options: /// starfish | starfish-mac | starfish-ml-dsa-44 | - /// starfish-speed | sparse-starfish-speed | + /// starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | + /// sparse-starfish-speed | /// starfish-bls | mysticeti | mysticeti-bls | /// cordial-miners | bluestreak | sailfish-pp #[clap( diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index cb9e44a3..f1f36aca 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -312,6 +312,14 @@ impl ProtocolConfig { ConsensusProtocol::Starfish, BlockAuthenticationScheme::MlDsa44, ), + "starfish-speed-mac" => ( + ConsensusProtocol::StarfishSpeed, + BlockAuthenticationScheme::MacVector, + ), + "starfish-speed-ml-dsa-44" => ( + ConsensusProtocol::StarfishSpeed, + BlockAuthenticationScheme::MlDsa44, + ), known => ( ConsensusProtocol::from_known_str(known) .ok_or_else(|| format!("Unknown consensus protocol '{known}'"))?, @@ -4945,6 +4953,28 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("starfish-speed").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::StarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::Ed25519, + } + ); + assert_eq!( + ProtocolConfig::from_str("starfish-speed-mac").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::StarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::MacVector, + } + ); + assert_eq!( + ProtocolConfig::from_str("starfish-speed-ml-dsa-44").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::StarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, + } + ); assert!(ProtocolConfig::from_str("starfish-unknown").is_err()); + assert!(ProtocolConfig::from_str("starfish-speed-unknown").is_err()); } } diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 23282285..0d43b2a4 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -2492,7 +2492,7 @@ mod tests { } #[test] - fn all_authentication_schemes_verify_for_starfish() { + fn all_authentication_schemes_verify_for_starfish_and_starfish_speed() { let committee = Committee::new_for_benchmarks(4); let ed_signers = Signer::new_for_test(committee.len()); let ml_dsa_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); @@ -2522,26 +2522,31 @@ mod tests { ), ]; - for (block, scheme) in cases { - for (receiver, receiver_keys) in mac_keyrings.iter().enumerate() { - let mut received = block.clone(); - let mut encoder = Encoder::new(2, 4, 2).unwrap(); - let mac_keys = if scheme == BlockAuthenticationScheme::MacVector { - receiver_keys.as_slice() - } else { - &[] - }; - received - .verify_with_authentication( - &committee, - receiver, - 0, - &mut encoder, - ConsensusProtocol::Starfish, - scheme, - mac_keys, - ) - .unwrap(); + for consensus_protocol in [ + ConsensusProtocol::Starfish, + ConsensusProtocol::StarfishSpeed, + ] { + for (block, scheme) in &cases { + for (receiver, receiver_keys) in mac_keyrings.iter().enumerate() { + let mut received = block.clone(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let mac_keys = if *scheme == BlockAuthenticationScheme::MacVector { + receiver_keys.as_slice() + } else { + &[] + }; + received + .verify_with_authentication( + &committee, + receiver, + 0, + &mut encoder, + consensus_protocol, + *scheme, + mac_keys, + ) + .unwrap(); + } } } } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 5a71a479..6dc238f0 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -344,6 +344,8 @@ mod smoke_tests { #[test_case("starfish-mac", 700)] #[test_case("starfish-ml-dsa-44", 720)] #[test_case("starfish-speed", 80)] + #[test_case("starfish-speed-mac", 760)] + #[test_case("starfish-speed-ml-dsa-44", 780)] #[test_case("starfish-bls", 100)] #[test_case("sailfish++", 120)] #[test_case("bluestreak", 140)] @@ -445,6 +447,8 @@ mod smoke_tests { #[test_case("starfish", 160)] #[test_case("starfish-mac", 740)] #[test_case("starfish-speed", 180)] + #[test_case("starfish-speed-mac", 800)] + #[test_case("starfish-speed-ml-dsa-44", 820)] #[test_case("starfish-bls", 200)] #[test_case("sailfish++", 220)] #[test_case("bluestreak", 260)] diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 9c8c6954..abf422cb 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -65,8 +65,8 @@ enum Operation { parameters_path: String, #[clap(long, value_name = "STRING", default_value = "")] byzantine_strategy: String, - /// Consensus/authentication variant (for example `starfish`, - /// `starfish-mac`, or `starfish-ml-dsa-44`). + /// Consensus/authentication variant (for example `starfish-mac` or + /// `starfish-speed-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, }, @@ -95,8 +95,8 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, - /// Consensus/authentication variant (for example `starfish`, - /// `starfish-mac`, or `starfish-ml-dsa-44`). + /// Consensus/authentication variant (for example `starfish-mac` or + /// `starfish-speed-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, /// Directory to store validator data (default: current directory) @@ -147,8 +147,8 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, - /// Consensus/authentication variant (for example `starfish`, - /// `starfish-mac`, or `starfish-ml-dsa-44`). + /// Consensus/authentication variant (for example `starfish-mac` or + /// `starfish-speed-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, #[clap(long, value_name = "INT", default_value_t = 600)] From 635550c6140e6f23303fe9d6b7c1dc5ea689708e Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:01:17 +0200 Subject: [PATCH 07/62] Fix local benchmark teardown and record auth comparison --- ...026-07-13-starfish-authentication-local.md | 84 +++++++++++++++++++ crates/starfish/src/main.rs | 30 +++---- 2 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 benchmark-results/2026-07-13-starfish-authentication-local.md diff --git a/benchmark-results/2026-07-13-starfish-authentication-local.md b/benchmark-results/2026-07-13-starfish-authentication-local.md new file mode 100644 index 00000000..2f8dbd88 --- /dev/null +++ b/benchmark-results/2026-07-13-starfish-authentication-local.md @@ -0,0 +1,84 @@ +# Starfish authentication comparison — local Apple Silicon + +Date: 2026-07-13 +Source revision: `92a3d85` plus the benchmark shutdown fix committed with this report +Host: Apple Silicon (`arm64`), macOS 15.7.4 +Build: Rust 1.86.0, release profile + +## Configuration + +- 10 honest validators in one local process +- 1,000 tx/s offered load (100 tx/s per validator) +- 20-second measurement window +- Uniform 0 ms added network latency +- Default protocol dissemination modes (`push-useful` for both families) +- One run per configuration + +Command template: + +```text +target/release/starfish local-benchmark \ + --committee-size 10 \ + --load 1000 \ + --consensus \ + --duration-secs 20 \ + --uniform-latency-ms 0 +``` + +## Results + +| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| Starfish | Ed25519 | 12.1 | 13.9 | 446.50 | 4,573.30 | 14.33 | 14.30 | 65.74 | +| Starfish | MAC vector | 12.8 | 14.8 | 433.60 | 4,494.05 | 14.04 | 14.01 | 66.34 | +| Starfish | ML-DSA-44 | 15.7 | 18.0 | 432.50 | 4,125.75 | 57.11 | 57.08 | 270.44 | +| Starfish Speed | Ed25519 | 10.2 | 12.3 | 442.75 | 4,201.30 | 14.61 | 14.58 | 67.60 | +| Starfish Speed | MAC vector | 9.4 | 11.4 | 439.15 | 4,114.10 | 14.12 | 14.09 | 65.85 | +| Starfish Speed | ML-DSA-44 | 13.4 | 17.4 | 453.80 | 2,868.60 | 39.84 | 39.82 | 179.82 | + +## Relative to Ed25519 within each protocol + +| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---|---:|---:|---:|---:|---:| +| Starfish | MAC vector | +5.8% | +6.5% | -2.9% | -1.7% | -2.0% | +| Starfish | ML-DSA-44 | +29.8% | +29.5% | -3.1% | -9.8% | +298.5% | +| Starfish Speed | MAC vector | -7.8% | -7.3% | -0.8% | -2.1% | -3.4% | +| Starfish Speed | ML-DSA-44 | +31.4% | +41.5% | +2.5% | -31.7% | +172.7% | + +## Starfish Speed relative to Starfish + +| Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---:|---:|---:|---:|---:| +| Ed25519 | -15.7% | -11.5% | -0.8% | -8.1% | +2.0% | +| MAC vector | -26.6% | -23.0% | +1.3% | -8.5% | +0.6% | +| ML-DSA-44 | -14.6% | -3.3% | +4.9% | -30.5% | -30.2% | + +## Interpretation + +- The MAC variants remained close to Ed25519: TPS was within 3%, BPS within + 2.1%, and bandwidth was slightly lower in this sample. The latency changes + are small enough that repeated runs are needed before treating their sign as + meaningful. +- ML-DSA-44 materially increased latency and bandwidth. Its signature is 2,420 + bytes, versus 64 bytes for Ed25519. Outbound bandwidth increased about 4.0x + for Starfish and 2.7x for Starfish Speed. +- Starfish Speed with ML-DSA-44 produced 31.7% fewer blocks than its Ed25519 + variant while committing 2.5% more transactions. This indicates more + transactions per block in this run; it should not be read as evidence that + ML-DSA improves throughput without repeated trials. +- Against matching Starfish authentication variants, Starfish Speed had lower + latency in all three samples and essentially equal TPS for Ed25519 and MAC. + Its ML-DSA-44 run used about 30% less bandwidth, alongside about 30% fewer + blocks, than Starfish ML-DSA-44. +- All variants achieved roughly 433–454 TPS from the 1,000 tx/s offered load. + Ten validators share one laptop and therefore contend for the same CPU, + storage, and network stack. These results are useful for directional local + comparison, not distributed capacity claims. + +## Benchmark harness fix + +The previous local benchmark shutdown aborted validator tasks and immediately +deleted their RocksDB directories. On macOS this left benchmark parents stuck +in an uninterruptible exiting state. The harness now uses a `JoinSet`, aborts +all validator tasks, drains them completely, and only then removes storage. +The validation run and all six measured runs exited normally. diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index abf422cb..5e03bb0d 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -28,6 +28,7 @@ use starfish_core::{ types::AuthorityIndex, validator::Validator, }; +use tokio::task::JoinSet; use tokio::time::Instant; use tracing_subscriber::{EnvFilter, filter::LevelFilter, fmt}; @@ -386,8 +387,7 @@ async fn local_benchmark( let base_dir = PathBuf::from("local-benchmark"); fs::create_dir_all(&base_dir)?; - let mut handles = Vec::with_capacity(committee_size); - let mut abort_handles = Vec::with_capacity(committee_size); + let mut validator_tasks = JoinSet::new(); let mut metrics_of_honest_validators = Vec::new(); let mut reporters_of_honest_validators = Vec::new(); @@ -459,12 +459,10 @@ async fn local_benchmark( } // Use the same pattern as the run method - let handle = tokio::spawn(async move { + validator_tasks.spawn(async move { let (network_result, _metrics_result) = validator.await_completion().await; network_result }); - abort_handles.push(handle.abort_handle()); - handles.push(handle); } // Run for specified duration @@ -481,23 +479,25 @@ async fn local_benchmark( duration_secs, ); - // Abort all tasks - for abort_handle in abort_handles { - abort_handle.abort(); + // Abort and fully drain validator tasks before deleting their + // RocksDB directories. On macOS, removing storage while aborted + // tasks are still dropping database handles can leave the + // benchmark parent stuck in an uninterruptible exit state. + validator_tasks.abort_all(); + while validator_tasks.join_next().await.is_some() { } // Clean up fs::remove_dir_all(base_dir)?; Ok(()) } - _ = async { - for handle in handles { - if let Err(e) = handle.await { - tracing::warn!("Validator terminated with error: {}", e); - } + result = validator_tasks.join_next() => { + running.store(false, Ordering::SeqCst); + tracing::warn!("Validator terminated before benchmark timeout: {result:?}"); + validator_tasks.abort_all(); + while validator_tasks.join_next().await.is_some() { } - } => { - println!("All validators completed before timeout"); + println!("A validator completed before timeout"); Metrics::aggregate_and_display( metrics_of_honest_validators, reporters_of_honest_validators, From 406423c9a4b0c3fb35fec5c87ce1dbb059df83b7 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:17:29 +0200 Subject: [PATCH 08/62] Add Sparse Starfish authentication variants --- README.md | 11 ++-- ...026-07-13-starfish-authentication-local.md | 45 +++++++++++--- crates/orchestrator/README.md | 2 +- crates/orchestrator/src/benchmark.rs | 3 +- crates/orchestrator/src/main.rs | 3 +- crates/starfish-core/src/broadcaster.rs | 48 +++------------ crates/starfish-core/src/dag_state.rs | 30 ++++++++++ crates/starfish-core/src/net_sync.rs | 57 ++++++++++++++++++ crates/starfish-core/src/types.rs | 58 +++++++++++-------- crates/starfish-core/src/validator.rs | 4 ++ crates/starfish/src/main.rs | 6 +- 11 files changed, 182 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index b0b655c8..f683627b 100644 --- a/README.md +++ b/README.md @@ -61,18 +61,19 @@ offloaded from the critical path. ### Starfish block authentication experiments -Plain Starfish and Starfish Speed can each be run with three interchangeable -block-authentication schemes: +Starfish, Starfish Speed, and Sparse-Starfish-Speed can each be run with three +interchangeable block-authentication schemes: | Protocol | Ed25519 | MAC vector | ML-DSA-44 | |---|---|---|---| | Starfish | `starfish` | `starfish-mac` | `starfish-ml-dsa-44` | | Starfish Speed | `starfish-speed` | `starfish-speed-mac` | `starfish-speed-ml-dsa-44` | +| Sparse-Starfish-Speed | `sparse-starfish-speed` | `sparse-starfish-speed-mac` | `sparse-starfish-speed-ml-dsa-44` | -For all six variants, `BlockReference.digest` is the BLAKE3 hash of the +For all nine variants, `BlockReference.digest` is the BLAKE3 hash of the canonical block content only. The authentication proof is a separate header -field and does not change the block reference. An author using either MAC -variant sends the full vector, with exactly one tag for every committee member, +field and does not change the block reference. An author using a MAC variant +sends the full vector, with exactly one tag for every committee member, to its direct recipients. A direct recipient retains that vector and, when relaying a header or answering a missing-parent request, sends only the destination's tag. A tag-only copy cannot be relayed a second time. Receivers diff --git a/benchmark-results/2026-07-13-starfish-authentication-local.md b/benchmark-results/2026-07-13-starfish-authentication-local.md index 2f8dbd88..e51ad802 100644 --- a/benchmark-results/2026-07-13-starfish-authentication-local.md +++ b/benchmark-results/2026-07-13-starfish-authentication-local.md @@ -1,7 +1,7 @@ # Starfish authentication comparison — local Apple Silicon Date: 2026-07-13 -Source revision: `92a3d85` plus the benchmark shutdown fix committed with this report +Source revision: `8a3bded` plus the Sparse authentication changes committed with this report
Host: Apple Silicon (`arm64`), macOS 15.7.4 Build: Rust 1.86.0, release profile @@ -11,7 +11,7 @@ Build: Rust 1.86.0, release profile - 1,000 tx/s offered load (100 tx/s per validator) - 20-second measurement window - Uniform 0 ms added network latency -- Default protocol dissemination modes (`push-useful` for both families) +- Default protocol dissemination mode (`push-useful` for all three protocols) - One run per configuration Command template: @@ -35,6 +35,9 @@ target/release/starfish local-benchmark \ | Starfish Speed | Ed25519 | 10.2 | 12.3 | 442.75 | 4,201.30 | 14.61 | 14.58 | 67.60 | | Starfish Speed | MAC vector | 9.4 | 11.4 | 439.15 | 4,114.10 | 14.12 | 14.09 | 65.85 | | Starfish Speed | ML-DSA-44 | 13.4 | 17.4 | 453.80 | 2,868.60 | 39.84 | 39.82 | 179.82 | +| Sparse-Starfish-Speed | Ed25519 | 5.5 | 10.8 | 430.00 | 4,867.50 | 8.99 | 8.96 | 42.84 | +| Sparse-Starfish-Speed | MAC vector | 5.2 | 10.5 | 430.75 | 5,211.95 | 9.73 | 9.69 | 46.25 | +| Sparse-Starfish-Speed | ML-DSA-44 | 5.5 | 10.1 | 440.05 | 4,335.90 | 37.85 | 37.82 | 176.16 | ## Relative to Ed25519 within each protocol @@ -44,6 +47,8 @@ target/release/starfish local-benchmark \ | Starfish | ML-DSA-44 | +29.8% | +29.5% | -3.1% | -9.8% | +298.5% | | Starfish Speed | MAC vector | -7.8% | -7.3% | -0.8% | -2.1% | -3.4% | | Starfish Speed | ML-DSA-44 | +31.4% | +41.5% | +2.5% | -31.7% | +172.7% | +| Sparse-Starfish-Speed | MAC vector | -5.5% | -2.8% | +0.2% | +7.1% | +8.2% | +| Sparse-Starfish-Speed | ML-DSA-44 | 0.0% | -6.5% | +2.3% | -10.9% | +321.0% | ## Starfish Speed relative to Starfish @@ -53,15 +58,23 @@ target/release/starfish local-benchmark \ | MAC vector | -26.6% | -23.0% | +1.3% | -8.5% | +0.6% | | ML-DSA-44 | -14.6% | -3.3% | +4.9% | -30.5% | -30.2% | +## Sparse-Starfish-Speed relative to Starfish + +| Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---:|---:|---:|---:|---:| +| Ed25519 | -54.5% | -22.3% | -3.7% | +6.4% | -37.3% | +| MAC vector | -59.4% | -29.1% | -0.7% | +16.0% | -30.7% | +| ML-DSA-44 | -65.0% | -43.9% | +1.7% | +5.1% | -33.7% | + ## Interpretation -- The MAC variants remained close to Ed25519: TPS was within 3%, BPS within - 2.1%, and bandwidth was slightly lower in this sample. The latency changes - are small enough that repeated runs are needed before treating their sign as - meaningful. +- The Starfish and Starfish Speed MAC variants remained close to Ed25519: TPS + was within 3%, BPS within 2.1%, and bandwidth was slightly lower in this + sample. The latency changes are small enough that repeated runs are needed + before treating their sign as meaningful. - ML-DSA-44 materially increased latency and bandwidth. Its signature is 2,420 bytes, versus 64 bytes for Ed25519. Outbound bandwidth increased about 4.0x - for Starfish and 2.7x for Starfish Speed. + for Starfish, 2.7x for Starfish Speed, and 4.2x for Sparse-Starfish-Speed. - Starfish Speed with ML-DSA-44 produced 31.7% fewer blocks than its Ed25519 variant while committing 2.5% more transactions. This indicates more transactions per block in this run; it should not be read as evidence that @@ -70,7 +83,14 @@ target/release/starfish local-benchmark \ latency in all three samples and essentially equal TPS for Ed25519 and MAC. Its ML-DSA-44 run used about 30% less bandwidth, alongside about 30% fewer blocks, than Starfish ML-DSA-44. -- All variants achieved roughly 433–454 TPS from the 1,000 tx/s offered load. +- Sparse-Starfish-Speed preserved roughly the same TPS as plain Starfish while + reducing outbound bandwidth by 31-37% and block latency by 55-65% across the + three authentication schemes. Its lean headers therefore remain beneficial + with either signatures or MACs in this local sample. +- Sparse MAC remained close to Sparse Ed25519: TPS differed by 0.2%, while + outbound bandwidth was 8.2% higher. Sparse ML-DSA-44 used 4.2x the outbound + bandwidth of Sparse Ed25519 despite Sparse's lower protocol overhead. +- All variants achieved roughly 430–454 TPS from the 1,000 tx/s offered load. Ten validators share one laptop and therefore contend for the same CPU, storage, and network stack. These results are useful for directional local comparison, not distributed capacity claims. @@ -81,4 +101,11 @@ The previous local benchmark shutdown aborted validator tasks and immediately deleted their RocksDB directories. On macOS this left benchmark parents stuck in an uninterruptible exiting state. The harness now uses a `JoinSet`, aborts all validator tasks, drains them completely, and only then removes storage. -The validation run and all six measured runs exited normally. +The validation run and all nine measured protocol/authentication combinations +exited normally. + +During the first Sparse MAC run, the receiver-side transport guard exposed a +round-gap response that still carried full MAC vectors. The sender now routes +round-gap blocks through the same recipient-tag preparation used by relay and +missing-parent paths. The rejected run was discarded; the Sparse MAC row above +is the clean rerun, which emitted no transport rejections. diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index db889f1f..1ae7d267 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -129,7 +129,7 @@ each load generator submits a fixed load of 100 tx/s or more precisely 10 tx every 100ms. Performance measurements are collected by regularly scraping the Prometheus metrics exposed by the load generators. -Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `sparse-starfish-speed`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. +Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, `sparse-starfish-speed-ml-dsa-44`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index dd5cfedd..374c0914 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -57,7 +57,8 @@ pub struct BenchmarkParametersGeneric { // Consensus protocol to deploy // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-speed | // starfish-speed-mac | starfish-speed-ml-dsa-44 | - // sparse-starfish-speed | starfish-bls | + // sparse-starfish-speed | sparse-starfish-speed-mac | + // sparse-starfish-speed-ml-dsa-44 | starfish-bls | // mysticeti | mysticeti-bls | cordial-miners | bluestreak | sailfish-pp) pub consensus_protocol: String, /// number Byzantine nodes diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index c42b062c..9f1bb685 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -137,7 +137,8 @@ pub enum Operation { /// Protocols to benchmark in order. Available options: /// starfish | starfish-mac | starfish-ml-dsa-44 | /// starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | - /// sparse-starfish-speed | + /// sparse-starfish-speed | sparse-starfish-speed-mac | + /// sparse-starfish-speed-ml-dsa-44 | /// starfish-bls | mysticeti | mysticeti-bls | /// cordial-miners | bluestreak | sailfish-pp #[clap( diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index b5f7a200..d1abc746 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -21,48 +21,16 @@ use crate::{ dag_state::{ByzantineStrategy, ConsensusProtocol, DataSource}, data::Data, metrics::{Metrics, UtilizationTimerVecExt}, - net_sync::NetworkSyncerInner, + net_sync::{NetworkSyncerInner, prepare_forwarded_blocks_for_peer}, network::{BlockBatch, NetworkMessage, ShardPayload}, runtime::{Handle, sleep}, syncer::CommitObserver, types::{ - AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockReference, RoundNumber, - VerifiedBlock, format_authority_index, + AuthorityIndex, AuthoritySet, BlockReference, RoundNumber, VerifiedBlock, + format_authority_index, }, }; -/// Prepare blocks sent through relay and missing-parent response paths for a -/// specific peer. In the MAC experiment, a direct recipient retains the -/// complete vector and selects only the destination's tag for these paths. A -/// tag-only copy cannot be relayed a second time and is therefore omitted. -fn prepare_relay_blocks_for_peer( - authentication_scheme: BlockAuthenticationScheme, - recipient: AuthorityIndex, - blocks: Vec>, -) -> Vec> { - if authentication_scheme != BlockAuthenticationScheme::MacVector { - return blocks; - } - - blocks - .into_iter() - .filter_map(|block| { - block - .with_recipient_mac(recipient) - .map(Data::new) - .or_else(|| { - tracing::debug!( - "Cannot relay MAC-authenticated block {} to authority {}: \ - complete MAC vector is unavailable", - block.reference(), - recipient, - ); - None - }) - }) - .collect() -} - fn peer_can_serve_missing_data( consensus_protocol: ConsensusProtocol, holders: &StakeAggregator, @@ -462,7 +430,7 @@ where .inner .dag_state .get_transmission_parts(&refs_to_send, &refs_to_send); - let headers = prepare_relay_blocks_for_peer( + let headers = prepare_forwarded_blocks_for_peer( self.inner.dag_state.block_authentication_scheme, peer_id, headers, @@ -1375,7 +1343,7 @@ where let (headers, shards) = inner .dag_state .get_transmission_parts(&plan.other_refs, &plan.shard_refs); - let headers = prepare_relay_blocks_for_peer( + let headers = prepare_forwarded_blocks_for_peer( inner.dag_state.block_authentication_scheme, to_whom_authority_index, headers, @@ -1572,7 +1540,7 @@ mod tests { use crate::{ committee::Committee, crypto::{SignatureBytes, mac_keyrings_for_test}, - types::{BaseTransaction, BlockAuthentication}, + types::{BaseTransaction, BlockAuthentication, BlockAuthenticationScheme}, }; fn holder_set(authorities: &[AuthorityIndex]) -> StakeAggregator { @@ -1649,7 +1617,7 @@ mod tests { let expected = tags[2]; block.header.authentication = BlockAuthentication::MacVector(tags); - let relayed = prepare_relay_blocks_for_peer( + let relayed = prepare_forwarded_blocks_for_peer( BlockAuthenticationScheme::MacVector, 2, vec![Data::new(block)], @@ -1661,7 +1629,7 @@ mod tests { )); let second_hop = - prepare_relay_blocks_for_peer(BlockAuthenticationScheme::MacVector, 3, relayed); + prepare_forwarded_blocks_for_peer(BlockAuthenticationScheme::MacVector, 3, relayed); assert!(second_hop.is_empty()); } } diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index f1f36aca..9929d6c6 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -320,6 +320,14 @@ impl ProtocolConfig { ConsensusProtocol::StarfishSpeed, BlockAuthenticationScheme::MlDsa44, ), + "sparse-starfish-speed-mac" => ( + ConsensusProtocol::SparseStarfishSpeed, + BlockAuthenticationScheme::MacVector, + ), + "sparse-starfish-speed-ml-dsa-44" => ( + ConsensusProtocol::SparseStarfishSpeed, + BlockAuthenticationScheme::MlDsa44, + ), known => ( ConsensusProtocol::from_known_str(known) .ok_or_else(|| format!("Unknown consensus protocol '{known}'"))?, @@ -4974,7 +4982,29 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("sparse-starfish-speed").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::Ed25519, + } + ); + assert_eq!( + ProtocolConfig::from_str("sparse-starfish-speed-mac").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::MacVector, + } + ); + assert_eq!( + ProtocolConfig::from_str("sparse-starfish-speed-ml-dsa-44").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, + } + ); assert!(ProtocolConfig::from_str("starfish-unknown").is_err()); assert!(ProtocolConfig::from_str("starfish-speed-unknown").is_err()); + assert!(ProtocolConfig::from_str("sparse-starfish-speed-unknown").is_err()); } } diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 5890fedf..4c05f936 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -98,6 +98,38 @@ fn verify_mac_transport( } } +/// Prepare blocks forwarded through relay or synchronization paths for a +/// specific peer. MAC-authenticated blocks retain their complete vector only +/// at direct recipients; forwarding selects the destination's tag. A +/// tag-only copy cannot be forwarded again and is therefore omitted. +pub(crate) fn prepare_forwarded_blocks_for_peer( + authentication_scheme: BlockAuthenticationScheme, + recipient: AuthorityIndex, + blocks: Vec>, +) -> Vec> { + if authentication_scheme != BlockAuthenticationScheme::MacVector { + return blocks; + } + + blocks + .into_iter() + .filter_map(|block| { + block + .with_recipient_mac(recipient) + .map(Data::new) + .or_else(|| { + tracing::debug!( + "Cannot forward MAC-authenticated block {} to authority {}: \ + complete MAC vector is unavailable", + block.reference(), + recipient, + ); + None + }) + }) + .collect() +} + async fn send_network_message_reliably( sender: &mpsc::Sender, message: NetworkMessage, @@ -1400,6 +1432,11 @@ impl ConnectionHandler Date: Mon, 13 Jul 2026 14:31:24 +0200 Subject: [PATCH 09/62] Record one-minute geographic auth benchmarks --- ...6-07-13-starfish-authentication-geo-60s.md | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 benchmark-results/2026-07-13-starfish-authentication-geo-60s.md diff --git a/benchmark-results/2026-07-13-starfish-authentication-geo-60s.md b/benchmark-results/2026-07-13-starfish-authentication-geo-60s.md new file mode 100644 index 00000000..a662738f --- /dev/null +++ b/benchmark-results/2026-07-13-starfish-authentication-geo-60s.md @@ -0,0 +1,102 @@ +# Starfish authentication comparison — 60-second geographic emulation + +Date: 2026-07-13
+Source revision: `d3f57c2`
+Host: Apple Silicon (`arm64`), macOS 15.7.4
+Build: Rust 1.86.0, release profile + +## Configuration + +- 10 honest validators in one local process +- 1,000 tx/s offered load (100 tx/s per validator) +- 60-second measurement window +- Default `push-useful` dissemination for all three protocols +- One run per configuration +- Geographic latency emulation enabled; no uniform-latency override + +The ten validators map in order to `us-east-1`, `us-west-1`, +`ca-central-1`, `eu-west-1`, `eu-south-1`, `eu-north-1`, `sa-east-1`, +`ap-south-1`, `ap-southeast-1`, and `ap-northeast-1`. The harness converts +its AWS RTT table to one-way delay by dividing each cell by two, then applies +independent ±3% per-message jitter. The resulting base one-way delays range +from 0.5 ms within a region to 154.5 ms between the most distant pair. + +This is a single-machine latency emulation, not a deployment on ten remote +hosts. CPU, storage, and the physical network stack remain shared. + +Command template: + +```text +target/release/starfish local-benchmark \ + --committee-size 10 \ + --load 1000 \ + --consensus \ + --duration-secs 60 +``` + +## Results + +| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| Starfish | Ed25519 | 553.4 | 603.7 | 816.25 | 100.05 | 0.49 | 0.49 | 1.23 | +| Starfish | MAC vector | 559.7 | 612.1 | 815.75 | 94.93 | 0.53 | 0.53 | 1.34 | +| Starfish | ML-DSA-44 | 554.1 | 604.2 | 814.70 | 97.00 | 0.76 | 0.76 | 1.92 | +| Starfish Speed | Ed25519 | 460.6 | 519.7 | 814.83 | 94.33 | 0.51 | 0.51 | 1.29 | +| Starfish Speed | MAC vector | 458.6 | 522.6 | 816.08 | 95.18 | 0.54 | 0.54 | 1.35 | +| Starfish Speed | ML-DSA-44 | 457.7 | 518.5 | 817.83 | 94.13 | 0.77 | 0.77 | 1.93 | +| Sparse-Starfish-Speed | Ed25519 | 417.6 | 485.1 | 818.25 | 95.42 | 0.45 | 0.45 | 1.13 | +| Sparse-Starfish-Speed | MAC vector | 418.7 | 483.5 | 816.25 | 92.70 | 0.47 | 0.47 | 1.18 | +| Sparse-Starfish-Speed | ML-DSA-44 | 420.3 | 486.9 | 817.95 | 92.85 | 0.71 | 0.71 | 1.78 | + +## Relative to Ed25519 within each protocol + +| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---|---:|---:|---:|---:|---:| +| Starfish | MAC vector | +1.1% | +1.4% | -0.1% | -5.1% | +8.2% | +| Starfish | ML-DSA-44 | +0.1% | +0.1% | -0.2% | -3.0% | +55.1% | +| Starfish Speed | MAC vector | -0.4% | +0.6% | +0.2% | +0.9% | +5.9% | +| Starfish Speed | ML-DSA-44 | -0.6% | -0.2% | +0.4% | -0.2% | +51.0% | +| Sparse-Starfish-Speed | MAC vector | +0.3% | -0.3% | -0.2% | -2.9% | +4.4% | +| Sparse-Starfish-Speed | ML-DSA-44 | +0.6% | +0.4% | 0.0% | -2.7% | +57.8% | + +## Protocol relative to matching Starfish authentication + +| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---|---:|---:|---:|---:|---:| +| Starfish Speed | Ed25519 | -16.8% | -13.9% | -0.2% | -5.7% | +4.1% | +| Starfish Speed | MAC vector | -18.1% | -14.6% | 0.0% | +0.3% | +1.9% | +| Starfish Speed | ML-DSA-44 | -17.4% | -14.2% | +0.4% | -3.0% | +1.3% | +| Sparse-Starfish-Speed | Ed25519 | -24.5% | -19.6% | +0.2% | -4.6% | -8.2% | +| Sparse-Starfish-Speed | MAC vector | -25.2% | -21.0% | +0.1% | -2.3% | -11.3% | +| Sparse-Starfish-Speed | ML-DSA-44 | -24.1% | -19.4% | +0.4% | -4.3% | -6.6% | + +## Interpretation + +- Authentication did not materially change throughput or latency under the + emulated wide-area delays. Within each protocol, TPS differed by at most + 0.4%, block latency by at most 1.1%, and end-to-end latency by at most 1.4%. + These small changes are below what should be interpreted without repeated + trials and variance estimates. +- MAC vectors increased outbound bandwidth by 4.4-8.2% relative to Ed25519, + while ML-DSA-44 increased it by 51.0-57.8%. At the geo-limited block rate, + payload and protocol traffic dominate more of the total than in the + zero-latency runs, so ML-DSA's relative bandwidth multiplier is smaller. +- Starfish Speed reduced block latency by 16.8-18.1% and end-to-end latency by + 13.9-14.6% against matching plain-Starfish authentication, with essentially + identical TPS. +- Sparse-Starfish-Speed reduced block latency by 24.1-25.2%, end-to-end + latency by 19.4-21.0%, and outbound bandwidth by 6.6-11.3% against matching + plain-Starfish authentication, again with essentially identical TPS. +- All variants committed roughly 815-818 TPS from the offered 1,000 tx/s. + This experiment measures a local machine under injected network delay; it + does not establish capacity on physically distributed hardware. + +## Caveats + +- There is one run per configuration and no warm-up exclusion, so these are + directional comparisons rather than confidence intervals. +- The reported metrics were emitted before shutdown. Validator task abortion + logs expected `JoinError::Cancelled` messages afterward. Two runs also + printed a macOS `pthread lock` teardown error after their metrics; process + and benchmark-directory checks showed no active or overlapping benchmark. + The shutdown path should still be hardened before unattended batch runs. From b219fea71ed1932b1e723ee4e11038f61e278b3c Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:40:30 +0200 Subject: [PATCH 10/62] Record local validator scaling limits --- .../2026-07-13-validator-scaling-probe.md | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 benchmark-results/2026-07-13-validator-scaling-probe.md diff --git a/benchmark-results/2026-07-13-validator-scaling-probe.md b/benchmark-results/2026-07-13-validator-scaling-probe.md new file mode 100644 index 00000000..0f722eb4 --- /dev/null +++ b/benchmark-results/2026-07-13-validator-scaling-probe.md @@ -0,0 +1,63 @@ +# Local validator-count scaling probe + +Date: 2026-07-13
+Source revision: `dee67ee`
+Host: Apple Silicon (`arm64`), macOS 15.7.4, 64 GiB RAM, 16 logical CPU cores + +## Purpose + +Find a practical committee-size limit for geographic latency emulation on one +machine. Offered load stays at 100 tx/s per validator. The 20-80-validator +probes run for 30 seconds under the AWS RTT latency model. The 10-validator +row is the earlier 60-second baseline and is included only for orientation. + +## Sparse-Starfish-Speed MAC results + +| Validators | Offered load | Duration | Block latency (ms) | E2E latency (ms) | TPS | BPS | Outbound (MB/s) | Outcome | +|---:|---:|---:|---:|---:|---:|---:|---:|---| +| 10 | 1,000 | 60 s | 418.7 | 483.5 | 816.25 | 92.70 | 0.47 | Clean baseline | +| 20 | 2,000 | 30 s | 435.85 | 501.05 | 1,240.83 | 191.67 | 0.88 | Clean | +| 40 | 4,000 | 30 s | 461.45 | 545.75 | 2,336.33 | 364.13 | 1.96 | Clean | +| 64 | 6,400 | 30 s | 480.42 | 540.67 | 3,435.23 | 591.83 | 3.47 | Clean, near resource saturation | +| 80 | 8,000 | 30 s | 662.36 | 752.25 | 2,238.30 | 470.77 | 2.86 | Unhealthy: socket-buffer and decode errors | + +During the active 64-validator run, the process reached about 1,032% CPU +(roughly 10 cores) and 17.7 GiB resident memory. At 80 validators the network +failed to establish and sustain the full mesh reliably: the run emitted +deserialization warnings and repeated macOS `No buffer space available` +errors. Its throughput regression and latency jump therefore mark it as an +invalid benchmark configuration on this host. + +## Common-size check with plain Starfish MAC + +| Validators | Offered load | Duration | Block latency (ms) | E2E latency (ms) | TPS | BPS | Outbound (MB/s) | Cancelled reconstructions | +|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| 40 | 4,000 | 30 s | 596.08 | 650.00 | 2,331.33 | 366.10 | 5.71 | 184 | + +Plain Starfish completed cleanly at 40 validators but used about 2.9x the +outbound bandwidth of Sparse-Starfish-Speed MAC at the same committee size. +It therefore provides the more conservative limit for a matrix that compares +all protocol families. + +## Limits and recommendation + +- The hard code limit is 512 validators (`MAX_COMMITTEE_SIZE`). This is a type + and data-structure bound, not a realistic single-machine target. +- The process file-descriptor limit is 1,048,575, and benchmark ports remain + well within `u16` even at 512 validators. Neither is the first constraint. +- The local network is a full mesh, so peer relationships grow as `n(n-1)`: + 1,560 at 40 validators, 4,032 at 64, and 6,320 at 80. Each validator also + owns a RocksDB instance. Socket buffers, connection tasks, and database + memory dominate before the hard committee or descriptor limits. +- Use 20 validators for quick, low-risk development comparisons. +- Use 40 validators as the recommended maximum for repeatable comparisons + across Starfish, Starfish Speed, Sparse, and all authentication schemes. +- Treat 64 as a Sparse-only local stress configuration, not a comfortable + full-matrix setting. +- Use multiple machines through the orchestrator beyond 40 validators. For + committees above ten, the local AWS table repeats the same ten regions while + all validators still share one kernel and physical host. + +The 30-second probes have different warm-up proportions from the 60-second +baseline, so their TPS values should not be used as a formal scaling curve. +The clean/error boundary and sampled resource usage are the relevant signals. From f4233db80466c4b86b4b99624a9a0a47d0f2e788 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:04:49 +0200 Subject: [PATCH 11/62] Record 40-validator geographic auth benchmarks --- ...sh-authentication-geo-40-validators-60s.md | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md diff --git a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md new file mode 100644 index 00000000..6ed5b47d --- /dev/null +++ b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md @@ -0,0 +1,126 @@ +# Starfish authentication comparison — 40-validator geographic emulation + +Date: 2026-07-13
+Source revision: `c90f8fb`
+Host: Apple Silicon (`arm64`), macOS 15.7.4
+Build: Rust 1.86.0, release profile + +## Configuration + +- 40 honest validators in one local process +- 1,000 tx/s aggregate offered load (25 tx/s per validator) +- 60-second measurement window +- Default `push-useful` dissemination for all three protocol families +- One run per configuration +- Geographic latency emulation enabled; no uniform-latency override + +The latency harness has ten AWS region profiles. At 40 validators, validator +indices are mapped modulo ten, producing four validators per modeled region. +The RTT values are divided by two to obtain one-way delays and independent +±3% per-message jitter is applied. Base one-way delays range from 0.5 ms to +154.5 ms. + +This is a single-machine latency emulation, not a deployment on 40 remote +hosts. All validators share the host's CPU, memory, storage, loopback network, +and kernel socket resources. + +In this report, **Sparse-Starfish-Speed means the sparse implementation of +Starfish Speed; it is not Bluestreak.** + +Command template: + +```text +target/release/starfish local-benchmark \ + --committee-size 40 \ + --load 1000 \ + --consensus \ + --duration-secs 60 +``` + +## Results + +| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| Starfish | Ed25519 | 590.92 | 644.60 | 790.63 | 407.02 | 5.28 | 5.27 | 13.67 | +| Starfish | MAC vector | 592.98 | 646.42 | 792.77 | 393.48 | 5.63 | 5.63 | 14.54 | +| Starfish | ML-DSA-44 | 571.38 | 627.02 | 792.33 | 399.95 | 7.44 | 7.44 | 19.22 | +| Starfish Speed | Ed25519 | 712.02 | 843.95 | 794.25 | 338.55 | 4.39 | 4.39 | 11.32 | +| Starfish Speed | MAC vector | 679.67 | 788.67 | 794.10 | 344.63 | 4.68 | 4.67 | 12.06 | +| Starfish Speed | ML-DSA-44 | 648.98 | 733.05 | 794.35 | 354.83 | 6.54 | 6.53 | 16.85 | +| Sparse-Starfish-Speed | Ed25519 | 446.98 | 507.93 | 795.08 | 367.70 | 0.85 | 0.85 | 2.19 | +| Sparse-Starfish-Speed | MAC vector | 441.60 | 502.15 | 790.58 | 366.55 | 1.23 | 1.23 | 3.19 | +| Sparse-Starfish-Speed | ML-DSA-44 | 435.23 | 499.48 | 794.33 | 371.28 | 2.38 | 2.38 | 6.14 | + +All nine commands exited successfully after printing their metrics. No +deserialize, socket-buffer, or transport errors were observed during these +runs. + +## Relative to Ed25519 within each protocol + +| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---|---:|---:|---:|---:|---:| +| Starfish | MAC vector | +0.3% | +0.3% | +0.3% | -3.3% | +6.6% | +| Starfish | ML-DSA-44 | -3.3% | -2.7% | +0.2% | -1.7% | +40.9% | +| Starfish Speed | MAC vector | -4.5% | -6.6% | 0.0% | +1.8% | +6.6% | +| Starfish Speed | ML-DSA-44 | -8.9% | -13.1% | 0.0% | +4.8% | +49.0% | +| Sparse-Starfish-Speed | MAC vector | -1.2% | -1.1% | -0.6% | -0.3% | +44.7% | +| Sparse-Starfish-Speed | ML-DSA-44 | -2.6% | -1.7% | -0.1% | +1.0% | +180.0% | + +The lower latency values in some MAC and ML-DSA runs must not be interpreted +as an authentication speedup. These are single, sequential trials without a +randomized order or variance estimates. + +## Protocol relative to matching Starfish authentication + +| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---|---:|---:|---:|---:|---:| +| Starfish Speed | Ed25519 | +20.5% | +30.9% | +0.5% | -16.8% | -16.9% | +| Starfish Speed | MAC vector | +14.6% | +22.0% | +0.2% | -12.4% | -16.9% | +| Starfish Speed | ML-DSA-44 | +13.6% | +16.9% | +0.3% | -11.3% | -12.1% | +| Sparse-Starfish-Speed | Ed25519 | -24.4% | -21.2% | +0.6% | -9.7% | -83.9% | +| Sparse-Starfish-Speed | MAC vector | -25.5% | -22.3% | -0.3% | -6.8% | -78.2% | +| Sparse-Starfish-Speed | ML-DSA-44 | -23.8% | -20.3% | +0.3% | -7.2% | -68.0% | + +## Interpretation + +- The corrected workload is 1,000 tx/s total, not 4,000 tx/s. The harness + divides the total evenly, so every validator generates 25 tx/s. +- Authentication choice did not materially affect committed throughput. All + variants reported 790.58-795.08 TPS, a spread of 0.6% across the complete + matrix. +- The local harness includes the 12-second connection warm-up in its + 60-second TPS denominator: the default 10 seconds plus 2 seconds for a + 40-validator committee. Roughly 48 seconds therefore submit transactions; + the measured 790.58-795.08 TPS corresponds to about 988-994 tx/s during the + active submission window, close to the offered 1,000 tx/s. +- For plain Starfish and Starfish Speed, MAC raises outbound bandwidth by + 6.6% over Ed25519. ML-DSA-44 raises it by 40.9% and 49.0%, respectively. +- Sparse-Starfish-Speed removes so much baseline protocol traffic that + authentication bytes become a larger fraction of the remainder. Its MAC + variant rises from 0.85 to 1.23 MB/s (+44.7%), and ML-DSA-44 rises to + 2.38 MB/s (+180.0%). The absolute traffic remains below every matching + non-sparse variant. +- The MAC result is consistent with the current design: direct author block + streaming carries the full committee-sized MAC vector, while relay and + synchronization paths carry one recipient tag. Consequently, the remaining + author-stream authentication cost grows with committee size. +- Sparse-Starfish-Speed is the strongest 40-validator result in this local + emulation: 435-447 ms block latency, 499-508 ms end-to-end latency, and + 0.85-2.38 MB/s outbound across the three authentication schemes. +- Starfish Speed alone was slower than plain Starfish at 40 validators even + though it was faster in the earlier 10-validator experiment. This reversal + points to a single-host scaling or run-variance effect and needs randomized, + repeated trials before it is treated as a protocol conclusion. + +## Caveats + +- There is one run per configuration and no randomized run order, warm-up + exclusion, or confidence interval. Relative authentication bandwidth is the + clearest result; latency differences need repeated trials. +- Four validators share each synthetic region profile. This produces the AWS + delay distribution but does not model independent machines or real WAN + bandwidth constraints. +- The 40 validators form a full mesh of 1,560 directed peer relationships and + use one RocksDB instance each. Host contention is part of the measurement. +- The progress window starts before transaction generation, which explains + why displayed TPS is below the aggregate offered rate. From eb0c20feb1410c8b2ef4cf729a41770733545ae6 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:12:40 +0200 Subject: [PATCH 12/62] Add Bluestreak authentication variants --- crates/starfish-core/src/broadcaster.rs | 22 ++++++++++++----- crates/starfish-core/src/dag_state.rs | 32 ++++++++++++++++++++++++- crates/starfish-core/src/net_sync.rs | 5 ++++ crates/starfish-core/src/validator.rs | 4 ++++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index d1abc746..8db012cd 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -471,6 +471,11 @@ where .into_iter() .flatten() .collect(); + let all_blocks = prepare_forwarded_blocks_for_peer( + self.inner.dag_state.block_authentication_scheme, + peer_id, + all_blocks, + ); let chunk_size = batch_block_size.max(1); // MissingParentsRequest responses must serve the entire requested @@ -1323,13 +1328,18 @@ where match push_transport_format(inner.dag_state.consensus_protocol) { PushOtherBlocksFormat::FullBlocks => { let mut full_blocks = plan.own_blocks; - full_blocks.extend( - inner - .dag_state - .get_transmission_blocks(&plan.other_refs) - .into_iter() - .flatten(), + let other_blocks = inner + .dag_state + .get_transmission_blocks(&plan.other_refs) + .into_iter() + .flatten() + .collect(); + let other_blocks = prepare_forwarded_blocks_for_peer( + inner.dag_state.block_authentication_scheme, + to_whom_authority_index, + other_blocks, ); + full_blocks.extend(other_blocks); BlockBatch { source: DataSource::BlockBundleStreaming, full_blocks, diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 9929d6c6..f724d892 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -328,6 +328,14 @@ impl ProtocolConfig { ConsensusProtocol::SparseStarfishSpeed, BlockAuthenticationScheme::MlDsa44, ), + "bluestreak-mac" => ( + ConsensusProtocol::Bluestreak, + BlockAuthenticationScheme::MacVector, + ), + "bluestreak-ml-dsa-44" => ( + ConsensusProtocol::Bluestreak, + BlockAuthenticationScheme::MlDsa44, + ), known => ( ConsensusProtocol::from_known_str(known) .ok_or_else(|| format!("Unknown consensus protocol '{known}'"))?, @@ -4939,7 +4947,7 @@ mod tests { } #[test] - fn protocol_config_selects_starfish_block_authentication() { + fn protocol_config_selects_block_authentication() { assert_eq!( ProtocolConfig::from_str("starfish").unwrap(), ProtocolConfig { @@ -5003,8 +5011,30 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("bluestreak").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Bluestreak, + block_authentication_scheme: BlockAuthenticationScheme::Ed25519, + } + ); + assert_eq!( + ProtocolConfig::from_str("bluestreak-mac").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Bluestreak, + block_authentication_scheme: BlockAuthenticationScheme::MacVector, + } + ); + assert_eq!( + ProtocolConfig::from_str("bluestreak-ml-dsa-44").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Bluestreak, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, + } + ); assert!(ProtocolConfig::from_str("starfish-unknown").is_err()); assert!(ProtocolConfig::from_str("starfish-speed-unknown").is_err()); assert!(ProtocolConfig::from_str("sparse-starfish-speed-unknown").is_err()); + assert!(ProtocolConfig::from_str("bluestreak-unknown").is_err()); } } diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 4c05f936..0b234c4c 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -1397,6 +1397,11 @@ impl ConnectionHandler Date: Tue, 14 Jul 2026 10:21:57 +0200 Subject: [PATCH 13/62] Document Bluestreak authentication benchmarks --- README.md | 32 +++++++++------- ...sh-authentication-geo-40-validators-60s.md | 38 ++++++++++++++----- crates/orchestrator/README.md | 2 +- crates/orchestrator/src/benchmark.rs | 5 ++- crates/orchestrator/src/main.rs | 3 +- crates/starfish/src/main.rs | 12 +++--- local-dryrun/README.md | 11 ++++-- local-dryrun/dryrun.sh | 7 ++-- 8 files changed, 71 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index f683627b..3f48750a 100644 --- a/README.md +++ b/README.md @@ -59,36 +59,42 @@ achieving 2-round optimistic commit latency. leader, data availability) in block headers, with async verification offloaded from the critical path. -### Starfish block authentication experiments +### Block authentication experiments -Starfish, Starfish Speed, and Sparse-Starfish-Speed can each be run with three -interchangeable block-authentication schemes: +Starfish, Starfish Speed, Sparse-Starfish-Speed, and Bluestreak can each be run +with three interchangeable block-authentication schemes: | Protocol | Ed25519 | MAC vector | ML-DSA-44 | |---|---|---|---| | Starfish | `starfish` | `starfish-mac` | `starfish-ml-dsa-44` | | Starfish Speed | `starfish-speed` | `starfish-speed-mac` | `starfish-speed-ml-dsa-44` | | Sparse-Starfish-Speed | `sparse-starfish-speed` | `sparse-starfish-speed-mac` | `sparse-starfish-speed-ml-dsa-44` | +| Bluestreak | `bluestreak` | `bluestreak-mac` | `bluestreak-ml-dsa-44` | -For all nine variants, `BlockReference.digest` is the BLAKE3 hash of the +For all twelve variants, `BlockReference.digest` is the BLAKE3 hash of the canonical block content only. The authentication proof is a separate header field and does not change the block reference. An author using a MAC variant sends the full vector, with exactly one tag for every committee member, to its direct recipients. A direct recipient retains that vector and, when -relaying a header or answering a missing-parent request, sends only the -destination's tag. A tag-only copy cannot be relayed a second time. Receivers -accept a full vector only through proactive block streaming directly from the -block's claimed author; relay and synchronization traffic must contain exactly -one recipient tag. If the same node later receives the author's directly -streamed full-vector copy, it upgrades the stored authentication without adding -a second DAG vertex and can then relay recipient-specific tags. Benchmark -genesis deterministically generates the pairwise MAC keys, ML-DSA seeds, and -public keys in the node configuration. +relaying a block or header, or answering a synchronization request, sends only +the destination's tag. A tag-only copy cannot be relayed a second time. +Receivers accept a full vector only through proactive block streaming directly +from the block's claimed author; relay and synchronization traffic must contain +exactly one recipient tag. If the same node later receives the author's +directly streamed full-vector copy, it upgrades the stored authentication +without adding a second DAG vertex and can then relay recipient-specific tags. +Benchmark genesis deterministically generates the pairwise MAC keys, ML-DSA +seeds, and public keys in the node configuration. This is research/benchmark code. The RustCrypto `ml-dsa` implementation used here states that it has not been independently audited and should not be treated as production-ready cryptography. +See the +[40-validator geographic authentication comparison](benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md) +for matching Ed25519, MAC-vector, and ML-DSA-44 measurements across all four +protocol families. + ## Dissemination Modes Every protocol can run with any of three dissemination strategies diff --git a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md index 6ed5b47d..697eb443 100644 --- a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md +++ b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md @@ -1,7 +1,7 @@ -# Starfish authentication comparison — 40-validator geographic emulation +# Starfish and Bluestreak authentication comparison — 40-validator geographic emulation -Date: 2026-07-13
-Source revision: `c90f8fb`
+Date: 2026-07-13–14
+Source revisions: `c90f8fb` (Starfish families), `3562890` (Bluestreak)
Host: Apple Silicon (`arm64`), macOS 15.7.4
Build: Rust 1.86.0, release profile @@ -10,7 +10,8 @@ Build: Rust 1.86.0, release profile - 40 honest validators in one local process - 1,000 tx/s aggregate offered load (25 tx/s per validator) - 60-second measurement window -- Default `push-useful` dissemination for all three protocol families +- Protocol-default dissemination: `push-useful` for the Starfish families and + `pull` for Bluestreak - One run per configuration - Geographic latency emulation enabled; no uniform-latency override @@ -50,8 +51,11 @@ target/release/starfish local-benchmark \ | Sparse-Starfish-Speed | Ed25519 | 446.98 | 507.93 | 795.08 | 367.70 | 0.85 | 0.85 | 2.19 | | Sparse-Starfish-Speed | MAC vector | 441.60 | 502.15 | 790.58 | 366.55 | 1.23 | 1.23 | 3.19 | | Sparse-Starfish-Speed | ML-DSA-44 | 435.23 | 499.48 | 794.33 | 371.28 | 2.38 | 2.38 | 6.14 | +| Bluestreak | Ed25519 | 417.65 | 477.93 | 793.48 | 389.53 | 0.59 | 0.59 | 1.53 | +| Bluestreak | MAC vector | 418.95 | 478.07 | 793.38 | 363.75 | 0.99 | 0.99 | 2.56 | +| Bluestreak | ML-DSA-44 | 420.70 | 478.43 | 795.43 | 368.57 | 1.41 | 1.41 | 3.62 | -All nine commands exited successfully after printing their metrics. No +All twelve commands exited successfully after printing their metrics. No deserialize, socket-buffer, or transport errors were observed during these runs. @@ -65,6 +69,8 @@ runs. | Starfish Speed | ML-DSA-44 | -8.9% | -13.1% | 0.0% | +4.8% | +49.0% | | Sparse-Starfish-Speed | MAC vector | -1.2% | -1.1% | -0.6% | -0.3% | +44.7% | | Sparse-Starfish-Speed | ML-DSA-44 | -2.6% | -1.7% | -0.1% | +1.0% | +180.0% | +| Bluestreak | MAC vector | +0.3% | 0.0% | 0.0% | -6.6% | +67.8% | +| Bluestreak | ML-DSA-44 | +0.7% | +0.1% | +0.2% | -5.4% | +139.0% | The lower latency values in some MAC and ML-DSA runs must not be interpreted as an authentication speedup. These are single, sequential trials without a @@ -80,21 +86,28 @@ randomized order or variance estimates. | Sparse-Starfish-Speed | Ed25519 | -24.4% | -21.2% | +0.6% | -9.7% | -83.9% | | Sparse-Starfish-Speed | MAC vector | -25.5% | -22.3% | -0.3% | -6.8% | -78.2% | | Sparse-Starfish-Speed | ML-DSA-44 | -23.8% | -20.3% | +0.3% | -7.2% | -68.0% | +| Bluestreak | Ed25519 | -29.3% | -25.9% | +0.4% | -4.3% | -88.8% | +| Bluestreak | MAC vector | -29.3% | -26.0% | +0.1% | -7.6% | -82.4% | +| Bluestreak | ML-DSA-44 | -26.4% | -23.7% | +0.4% | -7.8% | -81.0% | ## Interpretation - The corrected workload is 1,000 tx/s total, not 4,000 tx/s. The harness divides the total evenly, so every validator generates 25 tx/s. - Authentication choice did not materially affect committed throughput. All - variants reported 790.58-795.08 TPS, a spread of 0.6% across the complete + variants reported 790.58-795.43 TPS, a spread of 0.6% across the complete matrix. - The local harness includes the 12-second connection warm-up in its 60-second TPS denominator: the default 10 seconds plus 2 seconds for a 40-validator committee. Roughly 48 seconds therefore submit transactions; - the measured 790.58-795.08 TPS corresponds to about 988-994 tx/s during the + the measured 790.58-795.43 TPS corresponds to about 988-994 tx/s during the active submission window, close to the offered 1,000 tx/s. - For plain Starfish and Starfish Speed, MAC raises outbound bandwidth by 6.6% over Ed25519. ML-DSA-44 raises it by 40.9% and 49.0%, respectively. +- Bluestreak's baseline protocol traffic is even leaner. Its MAC vector raises + outbound bandwidth from 0.59 to 0.99 MB/s (+67.8%), while ML-DSA-44 raises + it to 1.41 MB/s (+139.0%). Latency and throughput remain within 0.7% of its + Ed25519 baseline. - Sparse-Starfish-Speed removes so much baseline protocol traffic that authentication bytes become a larger fraction of the remainder. Its MAC variant rises from 0.85 to 1.23 MB/s (+44.7%), and ML-DSA-44 rises to @@ -104,9 +117,12 @@ randomized order or variance estimates. streaming carries the full committee-sized MAC vector, while relay and synchronization paths carry one recipient tag. Consequently, the remaining author-stream authentication cost grows with committee size. -- Sparse-Starfish-Speed is the strongest 40-validator result in this local - emulation: 435-447 ms block latency, 499-508 ms end-to-end latency, and - 0.85-2.38 MB/s outbound across the three authentication schemes. +- Bluestreak is the strongest 40-validator result in this local emulation: + 418-421 ms block latency, about 478 ms end-to-end latency, and + 0.59-1.41 MB/s outbound across the three authentication schemes. Relative + to matching Sparse-Starfish-Speed authentication, it reduces block latency + by 3.3-6.6%, end-to-end latency by 4.2-5.9%, and outbound bandwidth by + 19.5-40.8%. - Starfish Speed alone was slower than plain Starfish at 40 validators even though it was faster in the earlier 10-validator experiment. This reversal points to a single-host scaling or run-variance effect and needs randomized, @@ -117,6 +133,8 @@ randomized order or variance estimates. - There is one run per configuration and no randomized run order, warm-up exclusion, or confidence interval. Relative authentication bandwidth is the clearest result; latency differences need repeated trials. +- The Bluestreak runs were made one day after the Starfish-family runs using + the same host and benchmark configuration but a newer source revision. - Four validators share each synthetic region profile. This produces the AWS delay distribution but does not model independent machines or real WAN bandwidth constraints. diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index 1ae7d267..02247b48 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -129,7 +129,7 @@ each load generator submits a fixed load of 100 tx/s or more precisely 10 tx every 100ms. Performance measurements are collected by regularly scraping the Prometheus metrics exposed by the load generators. -Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, `sparse-starfish-speed-ml-dsa-44`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `bluestreak`, `cordial-miners`, `sailfish-pp`. +Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, `sparse-starfish-speed-ml-dsa-44`, `bluestreak`, `bluestreak-mac`, `bluestreak-ml-dsa-44`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `cordial-miners`, `sailfish-pp`. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index 374c0914..e9ebd8f0 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -58,8 +58,9 @@ pub struct BenchmarkParametersGeneric { // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-speed | // starfish-speed-mac | starfish-speed-ml-dsa-44 | // sparse-starfish-speed | sparse-starfish-speed-mac | - // sparse-starfish-speed-ml-dsa-44 | starfish-bls | - // mysticeti | mysticeti-bls | cordial-miners | bluestreak | sailfish-pp) + // sparse-starfish-speed-ml-dsa-44 | bluestreak | bluestreak-mac | + // bluestreak-ml-dsa-44 | starfish-bls | mysticeti | mysticeti-bls | + // cordial-miners | sailfish-pp) pub consensus_protocol: String, /// number Byzantine nodes pub byzantine_nodes: usize, diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 9f1bb685..9e4811be 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -139,8 +139,9 @@ pub enum Operation { /// starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | /// sparse-starfish-speed | sparse-starfish-speed-mac | /// sparse-starfish-speed-ml-dsa-44 | + /// bluestreak | bluestreak-mac | bluestreak-ml-dsa-44 | /// starfish-bls | mysticeti | mysticeti-bls | - /// cordial-miners | bluestreak | sailfish-pp + /// cordial-miners | sailfish-pp #[clap( long, value_name = "STRING", diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 0206a5ec..12b3cef9 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -66,8 +66,8 @@ enum Operation { parameters_path: String, #[clap(long, value_name = "STRING", default_value = "")] byzantine_strategy: String, - /// Consensus/authentication variant (for example `starfish-mac` or - /// `sparse-starfish-speed-ml-dsa-44`). + /// Consensus/authentication variant (for example `starfish-mac`, + /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, }, @@ -96,8 +96,8 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, - /// Consensus/authentication variant (for example `starfish-mac` or - /// `sparse-starfish-speed-ml-dsa-44`). + /// Consensus/authentication variant (for example `starfish-mac`, + /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, /// Directory to store validator data (default: current directory) @@ -148,8 +148,8 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, - /// Consensus/authentication variant (for example `starfish-mac` or - /// `sparse-starfish-speed-ml-dsa-44`). + /// Consensus/authentication variant (for example `starfish-mac`, + /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-44`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, #[clap(long, value_name = "INT", default_value_t = 600)] diff --git a/local-dryrun/README.md b/local-dryrun/README.md index 390d43cb..f2c54f69 100644 --- a/local-dryrun/README.md +++ b/local-dryrun/README.md @@ -32,9 +32,14 @@ NUM_NODES=10 DESIRED_TPS=1000 CONSENSUS=starfish \ | `CONSENSUS` | `bluestreak` | Consensus protocol (see below) | | `TEST_TIME` | `3000` | Experiment duration in seconds | -Supported `CONSENSUS` values: `starfish`, `starfish-speed`, -`sparse-starfish-speed`, `starfish-bls`, `cordial-miners`, -`mysticeti`, `sailfish-pp`, `bluestreak`, `mysticeti-bls`. +Supported `CONSENSUS` values include the Ed25519, MAC, and ML-DSA-44 variants +of Starfish, Starfish Speed, Sparse-Starfish-Speed, and Bluestreak: +`starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, +`starfish-speed-mac`, `starfish-speed-ml-dsa-44`, +`sparse-starfish-speed`, `sparse-starfish-speed-mac`, +`sparse-starfish-speed-ml-dsa-44`, `bluestreak`, `bluestreak-mac`, +`bluestreak-ml-dsa-44`, `starfish-bls`, `cordial-miners`, `mysticeti`, +`sailfish-pp`, and `mysticeti-bls`. ### Protocol Tuning diff --git a/local-dryrun/dryrun.sh b/local-dryrun/dryrun.sh index b4ed06a7..37982a4c 100755 --- a/local-dryrun/dryrun.sh +++ b/local-dryrun/dryrun.sh @@ -6,9 +6,10 @@ NUM_NODES=${NUM_NODES:-10} NUM_CRASHED_NODES=${NUM_CRASHED_NODES:-0} DESIRED_TPS=${DESIRED_TPS:-100} -# Options: starfish, starfish-speed, sparse-starfish-speed, -# starfish-bls, cordial-miners, mysticeti, sailfish-pp, -# bluestreak, mysticeti-bls +# Authentication variants append -mac or -ml-dsa-44 to starfish, +# starfish-speed, sparse-starfish-speed, or bluestreak. +# Other options: starfish-bls, cordial-miners, mysticeti, +# sailfish-pp, mysticeti-bls CONSENSUS=${CONSENSUS:- sparse-starfish-speed} NUM_BYZANTINE_NODES=${NUM_BYZANTINE_NODES:-0} # Options: timeout-leader, leader-withholding, From a8f3697d7eba8b7cff8c71d88cdb9a774961f8d3 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:56:58 +0200 Subject: [PATCH 14/62] Record 60-validator authentication benchmarks --- README.md | 4 +- ...14-authentication-geo-60-validators-60s.md | 138 ++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md diff --git a/README.md b/README.md index 3f48750a..572e495f 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,9 @@ treated as production-ready cryptography. See the [40-validator geographic authentication comparison](benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md) for matching Ed25519, MAC-vector, and ML-DSA-44 measurements across all four -protocol families. +protocol families. A separate +[60-validator moderate-load comparison](benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md) +records the single-machine scaling boundary at 600 tx/s aggregate load. ## Dissemination Modes diff --git a/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md b/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md new file mode 100644 index 00000000..3182c901 --- /dev/null +++ b/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md @@ -0,0 +1,138 @@ +# Authentication comparison — 60-validator geographic emulation + +Date: 2026-07-14
+Source revision: `4553e5e`
+Host: Apple Silicon (`arm64`), macOS 15.7.4
+Build: Rust 1.86.0, release profile + +## Configuration + +- 60 honest validators in one local process +- 600 tx/s aggregate offered load (exactly 10 tx/s per validator) +- 60-second measurement window +- Protocol-default dissemination: `push-useful` for the Starfish families and + `pull` for Bluestreak +- One run per configuration +- Geographic latency emulation enabled; no uniform-latency override + +The harness has ten AWS region profiles. Validator indices map to profiles +modulo ten, so this experiment models six validators per region. RTT values +are divided by two to obtain one-way delays and independent ±3% per-message +jitter is applied. Base one-way delays range from 0.5 ms to 154.5 ms. + +This is a single-machine latency emulation, not a 60-host deployment. The +validators share CPU, memory, storage, loopback networking, and kernel socket +resources. They form 3,540 directed peer relationships and use one RocksDB +instance each. + +Command template: + +```text +target/release/starfish local-benchmark \ + --committee-size 60 \ + --load 600 \ + --consensus \ + --duration-secs 60 +``` + +## Results + +| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| Starfish | Ed25519 | 664.45 | 728.77 | 444.30 | 515.02 | 8.59 | 8.58 | 39.59 | +| Starfish | MAC vector | 680.68 | 753.42 | 414.87 | 449.23 | 8.35 | 8.34 | 41.21 | +| Starfish | ML-DSA-44 | 849.38 | 937.02 | 427.77 | 417.95 | 9.72 | 9.71 | 46.53 | +| Starfish Speed | Ed25519 | 592.13 | 673.87 | 460.37 | 467.12 | 6.63 | 6.62 | 29.49 | +| Starfish Speed | MAC vector | 670.80 | 783.93 | 430.95 | 439.30 | 7.30 | 7.30 | 34.70 | +| Starfish Speed | ML-DSA-44 | 641.70 | 726.03 | 433.63 | 461.57 | 11.06 | 11.05 | 52.22 | +| Sparse-Starfish-Speed | Ed25519 | 467.70 | 525.98 | 466.22 | 535.57 | 0.88 | 0.88 | 3.87 | +| Sparse-Starfish-Speed | MAC vector | 467.67 | 528.23 | 466.62 | 551.88 | 1.84 | 1.84 | 8.08 | +| Sparse-Starfish-Speed | ML-DSA-44 | 464.87 | 526.97 | 467.07 | 528.88 | 2.98 | 2.98 | 13.08 | +| Bluestreak | Ed25519 | 422.53 | 480.58 | 468.22 | 551.08 | 0.52 | 0.52 | 2.28 | +| Bluestreak | MAC vector | 421.62 | 480.37 | 467.63 | 545.72 | 1.47 | 1.46 | 6.42 | +| Bluestreak | ML-DSA-44 | 422.72 | 481.70 | 467.88 | 547.13 | 1.75 | 1.75 | 7.67 | + +All twelve commands exited successfully after emitting their metrics. No +authentication, deserialize, socket-buffer, or transport errors were observed. + +## Warm-up-adjusted offered-rate utilization + +The transaction generators wait 13 seconds before submitting: the default +10-second delay plus 3 seconds for a 60-validator committee. The displayed TPS +uses the complete 60-second window, leaving approximately 47 active submission +seconds. The following values estimate the active rate as `TPS × 60 / 47`. + +| Protocol | Authentication | Estimated active TPS | Offered rate sustained | +|---|---|---:|---:| +| Starfish | Ed25519 | 567.19 | 94.5% | +| Starfish | MAC vector | 529.62 | 88.3% | +| Starfish | ML-DSA-44 | 546.09 | 91.0% | +| Starfish Speed | Ed25519 | 587.71 | 98.0% | +| Starfish Speed | MAC vector | 550.15 | 91.7% | +| Starfish Speed | ML-DSA-44 | 553.57 | 92.3% | +| Sparse-Starfish-Speed | Ed25519 | 595.17 | 99.2% | +| Sparse-Starfish-Speed | MAC vector | 595.69 | 99.3% | +| Sparse-Starfish-Speed | ML-DSA-44 | 596.26 | 99.4% | +| Bluestreak | Ed25519 | 597.73 | 99.6% | +| Bluestreak | MAC vector | 596.97 | 99.5% | +| Bluestreak | ML-DSA-44 | 597.29 | 99.5% | + +## Relative to Ed25519 within each protocol + +| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | +|---|---|---:|---:|---:|---:|---:| +| Starfish | MAC vector | +2.4% | +3.4% | -6.6% | -12.8% | -2.8% | +| Starfish | ML-DSA-44 | +27.8% | +28.6% | -3.7% | -18.8% | +13.2% | +| Starfish Speed | MAC vector | +13.3% | +16.3% | -6.4% | -6.0% | +10.1% | +| Starfish Speed | ML-DSA-44 | +8.4% | +7.7% | -5.8% | -1.2% | +66.8% | +| Sparse-Starfish-Speed | MAC vector | 0.0% | +0.4% | +0.1% | +3.0% | +109.1% | +| Sparse-Starfish-Speed | ML-DSA-44 | -0.6% | +0.2% | +0.2% | -1.2% | +238.6% | +| Bluestreak | MAC vector | -0.2% | 0.0% | -0.1% | -1.0% | +182.7% | +| Bluestreak | ML-DSA-44 | 0.0% | +0.2% | -0.1% | -0.7% | +236.5% | + +Raw bandwidth can fall despite a larger authentication proof when a run +produces fewer blocks, as in plain Starfish MAC. The bandwidth-efficiency +metric—the ratio of bytes sent to committed transaction-payload bytes—rises +from 39.59 to 41.21 and captures the normalized increase. + +## Interpretation + +- Sixty validators are feasible on this machine at a 600 tx/s aggregate + offered load, but headroom depends strongly on the protocol family. +- Sparse-Starfish-Speed and Bluestreak sustain 99.2-99.6% of the active offered + rate for every authentication scheme. Their latency and throughput vary by + at most 0.6% and 0.2%, respectively, within each family. +- Plain Starfish sustains 88.3-94.5% of the active offered rate, while + Starfish Speed sustains 91.7-98.0%. Their authentication comparisons are + therefore partly measurements of shared-host contention. In particular, + plain Starfish ML-DSA-44 has 27.8% higher block latency than its Ed25519 run. +- Bluestreak has the lowest latency and absolute bandwidth across every + authentication scheme: 422-423 ms block latency, 480-482 ms end-to-end + latency, and 0.52-1.75 MB/s outbound. +- Relative to matching Sparse-Starfish-Speed authentication, Bluestreak lowers + block latency by 9.1-9.8%, end-to-end latency by 8.6-9.1%, and outbound + bandwidth by 20.1-41.3%, with nearly identical throughput. +- At 60 validators, a full author MAC vector contains 60 × 32 = 1,920 bytes, + compared with a 64-byte Ed25519 signature and a 2,420-byte ML-DSA-44 + signature. Relays and synchronization responses still carry only one + 32-byte recipient tag. Because Bluestreak and Sparse-Starfish-Speed remove + most other traffic, these authentication bytes produce large percentages + while their absolute bandwidth remains well below the denser protocols. +- For repeatable all-protocol authentication comparisons on this single host, + 40 validators remains the safer configuration. Sixty validators is a useful + stress configuration and a clean operating point for the sparse and + Bluestreak families; denser families should move to multiple machines for + stronger conclusions. + +## Caveats + +- There is one sequential run per configuration, with no randomized order or + confidence interval. Host scheduling and thermal state can affect results. +- The 600 tx/s load differs from the earlier 40-validator 1,000 tx/s matrix; + the two experiments should not be treated as a pure committee-size scaling + comparison. +- Six validators share each synthetic AWS region profile. The harness injects + the delay distribution but does not model independent machines or WAN + bandwidth constraints. +- Displayed TPS includes the 13-second startup delay. Warm-up-adjusted TPS is + derived rather than measured in a separately gated metrics window. From 8f5072989e7b98f190bbec5141130e046738cb12 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:48:21 +0200 Subject: [PATCH 15/62] Add ML-DSA-65 authentication variants --- README.md | 20 +- crates/orchestrator/README.md | 2 +- crates/orchestrator/src/benchmark.rs | 10 +- crates/orchestrator/src/main.rs | 8 +- crates/starfish-core/src/committee.rs | 38 ++- crates/starfish-core/src/config.rs | 18 +- crates/starfish-core/src/core.rs | 7 +- crates/starfish-core/src/crypto.rs | 349 ++++++++++++++++---------- crates/starfish-core/src/dag_state.rs | 92 ++++--- crates/starfish-core/src/types.rs | 45 +++- crates/starfish-core/src/validator.rs | 18 ++ crates/starfish/src/main.rs | 6 +- local-dryrun/README.md | 12 +- local-dryrun/dryrun.sh | 2 +- 14 files changed, 410 insertions(+), 217 deletions(-) diff --git a/README.md b/README.md index 572e495f..8d844c37 100644 --- a/README.md +++ b/README.md @@ -62,16 +62,16 @@ offloaded from the critical path. ### Block authentication experiments Starfish, Starfish Speed, Sparse-Starfish-Speed, and Bluestreak can each be run -with three interchangeable block-authentication schemes: +with four interchangeable block-authentication schemes: -| Protocol | Ed25519 | MAC vector | ML-DSA-44 | -|---|---|---|---| -| Starfish | `starfish` | `starfish-mac` | `starfish-ml-dsa-44` | -| Starfish Speed | `starfish-speed` | `starfish-speed-mac` | `starfish-speed-ml-dsa-44` | -| Sparse-Starfish-Speed | `sparse-starfish-speed` | `sparse-starfish-speed-mac` | `sparse-starfish-speed-ml-dsa-44` | -| Bluestreak | `bluestreak` | `bluestreak-mac` | `bluestreak-ml-dsa-44` | +| Protocol | Ed25519 | MAC vector | ML-DSA-44 | ML-DSA-65 | +|---|---|---|---|---| +| Starfish | `starfish` | `starfish-mac` | `starfish-ml-dsa-44` | `starfish-ml-dsa-65` | +| Starfish Speed | `starfish-speed` | `starfish-speed-mac` | `starfish-speed-ml-dsa-44` | `starfish-speed-ml-dsa-65` | +| Sparse-Starfish-Speed | `sparse-starfish-speed` | `sparse-starfish-speed-mac` | `sparse-starfish-speed-ml-dsa-44` | `sparse-starfish-speed-ml-dsa-65` | +| Bluestreak | `bluestreak` | `bluestreak-mac` | `bluestreak-ml-dsa-44` | `bluestreak-ml-dsa-65` | -For all twelve variants, `BlockReference.digest` is the BLAKE3 hash of the +For all sixteen variants, `BlockReference.digest` is the BLAKE3 hash of the canonical block content only. The authentication proof is a separate header field and does not change the block reference. An author using a MAC variant sends the full vector, with exactly one tag for every committee member, @@ -86,6 +86,10 @@ without adding a second DAG vertex and can then relay recipient-specific tags. Benchmark genesis deterministically generates the pairwise MAC keys, ML-DSA seeds, and public keys in the node configuration. +The ML-DSA wrappers are generated from a common parameter-set definition. +ML-DSA-44 uses 1,312-byte public keys and 2,420-byte signatures; ML-DSA-65 +uses 1,952-byte public keys and 3,309-byte signatures. + This is research/benchmark code. The RustCrypto `ml-dsa` implementation used here states that it has not been independently audited and should not be treated as production-ready cryptography. diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index 02247b48..2dbf4077 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -129,7 +129,7 @@ each load generator submits a fixed load of 100 tx/s or more precisely 10 tx every 100ms. Performance measurements are collected by regularly scraping the Prometheus metrics exposed by the load generators. -Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, `sparse-starfish-speed-ml-dsa-44`, `bluestreak`, `bluestreak-mac`, `bluestreak-ml-dsa-44`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `cordial-miners`, `sailfish-pp`. +Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-ml-dsa-65`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `starfish-speed-ml-dsa-65`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, `sparse-starfish-speed-ml-dsa-44`, `sparse-starfish-speed-ml-dsa-65`, `bluestreak`, `bluestreak-mac`, `bluestreak-ml-dsa-44`, `bluestreak-ml-dsa-65`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `cordial-miners`, `sailfish-pp`. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index e9ebd8f0..68487b56 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -55,11 +55,13 @@ pub struct BenchmarkParametersGeneric { /// paying for data sent between the nodes. pub use_internal_ip_address: bool, // Consensus protocol to deploy - // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-speed | - // starfish-speed-mac | starfish-speed-ml-dsa-44 | + // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-ml-dsa-65 | + // starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | + // starfish-speed-ml-dsa-65 | // sparse-starfish-speed | sparse-starfish-speed-mac | - // sparse-starfish-speed-ml-dsa-44 | bluestreak | bluestreak-mac | - // bluestreak-ml-dsa-44 | starfish-bls | mysticeti | mysticeti-bls | + // sparse-starfish-speed-ml-dsa-44 | sparse-starfish-speed-ml-dsa-65 | + // bluestreak | bluestreak-mac | bluestreak-ml-dsa-44 | + // bluestreak-ml-dsa-65 | starfish-bls | mysticeti | mysticeti-bls | // cordial-miners | sailfish-pp) pub consensus_protocol: String, /// number Byzantine nodes diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 9e4811be..fc2cf3f6 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -135,11 +135,11 @@ pub enum Operation { skip_testbed_configuration: bool, /// Protocols to benchmark in order. Available options: - /// starfish | starfish-mac | starfish-ml-dsa-44 | - /// starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | + /// starfish | starfish-mac | starfish-ml-dsa-44 | starfish-ml-dsa-65 | + /// starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | starfish-speed-ml-dsa-65 | /// sparse-starfish-speed | sparse-starfish-speed-mac | - /// sparse-starfish-speed-ml-dsa-44 | - /// bluestreak | bluestreak-mac | bluestreak-ml-dsa-44 | + /// sparse-starfish-speed-ml-dsa-44 | sparse-starfish-speed-ml-dsa-65 | + /// bluestreak | bluestreak-mac | bluestreak-ml-dsa-44 | bluestreak-ml-dsa-65 | /// starfish-bls | mysticeti | mysticeti-bls | /// cordial-miners | sailfish-pp #[clap( diff --git a/crates/starfish-core/src/committee.rs b/crates/starfish-core/src/committee.rs index d6515928..8518d758 100644 --- a/crates/starfish-core/src/committee.rs +++ b/crates/starfish-core/src/committee.rs @@ -11,8 +11,9 @@ use serde::{Deserialize, Serialize}; use crate::{ config::ImportExport, crypto::{ - BlsPublicKey, BlsSigner, MlDsa44PublicKey, MlDsa44Signer, PublicKey, Signer, - dummy_bls_public_key, dummy_ml_dsa_44_public_key, dummy_public_key, + BlsPublicKey, BlsSigner, MlDsa44PublicKey, MlDsa44Signer, MlDsa65PublicKey, MlDsa65Signer, + PublicKey, Signer, dummy_bls_public_key, dummy_ml_dsa_44_public_key, + dummy_ml_dsa_65_public_key, dummy_public_key, }, data::Data, types::{AuthorityIndex, AuthoritySet, RoundNumber, Stake, VerifiedBlock}, @@ -152,6 +153,12 @@ impl Committee { .map(Authority::ml_dsa_44_public_key) } + pub fn get_ml_dsa_65_public_key(&self, authority: AuthorityIndex) -> Option<&MlDsa65PublicKey> { + self.authorities + .get(authority as usize) + .map(Authority::ml_dsa_65_public_key) + } + pub fn known_authority(&self, authority: AuthorityIndex) -> bool { (authority as usize) < self.len() } @@ -215,18 +222,23 @@ impl Committee { pub fn new_for_benchmarks(committee_size: usize) -> Arc { let signers = Signer::new_for_test(committee_size); let bls_signers = BlsSigner::new_for_test(committee_size); - let ml_dsa_signers = MlDsa44Signer::new_for_test(committee_size); + let ml_dsa_44_signers = MlDsa44Signer::new_for_test(committee_size); + let ml_dsa_65_signers = MlDsa65Signer::new_for_test(committee_size); Self::new( signers .into_iter() .zip(bls_signers) - .zip(ml_dsa_signers) - .map(|((keypair, bls_keypair), ml_dsa_keypair)| Authority { - stake: 1, - public_key: keypair.public_key(), - bls_public_key: bls_keypair.public_key(), - ml_dsa_44_public_key: ml_dsa_keypair.public_key(), - }) + .zip(ml_dsa_44_signers) + .zip(ml_dsa_65_signers) + .map( + |(((keypair, bls_keypair), ml_dsa_44_keypair), ml_dsa_65_keypair)| Authority { + stake: 1, + public_key: keypair.public_key(), + bls_public_key: bls_keypair.public_key(), + ml_dsa_44_public_key: ml_dsa_44_keypair.public_key(), + ml_dsa_65_public_key: ml_dsa_65_keypair.public_key(), + }, + ) .collect(), ) } @@ -238,6 +250,7 @@ pub struct Authority { public_key: PublicKey, bls_public_key: BlsPublicKey, ml_dsa_44_public_key: MlDsa44PublicKey, + ml_dsa_65_public_key: MlDsa65PublicKey, } impl Authority { @@ -247,6 +260,7 @@ impl Authority { public_key: dummy_public_key(), bls_public_key: dummy_bls_public_key(), ml_dsa_44_public_key: dummy_ml_dsa_44_public_key(), + ml_dsa_65_public_key: dummy_ml_dsa_65_public_key(), } } @@ -265,6 +279,10 @@ impl Authority { pub fn ml_dsa_44_public_key(&self) -> &MlDsa44PublicKey { &self.ml_dsa_44_public_key } + + pub fn ml_dsa_65_public_key(&self) -> &MlDsa65PublicKey { + &self.ml_dsa_65_public_key + } } impl ImportExport for Committee {} diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 05758057..5ac24437 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -13,8 +13,8 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ crypto::{ - BlsPublicKey, BlsSigner, MacKey, MlDsa44Signer, Signer, dummy_bls_signer, - dummy_ml_dsa_44_signer, dummy_signer, mac_keyrings_for_test, + BlsPublicKey, BlsSigner, MacKey, MlDsa44Signer, MlDsa65Signer, Signer, dummy_bls_signer, + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, types::{AuthorityIndex, PublicKey, RoundNumber}, }; @@ -274,6 +274,7 @@ pub struct NodePrivateConfig { pub keypair: Signer, pub bls_keypair: BlsSigner, pub ml_dsa_44_keypair: MlDsa44Signer, + pub ml_dsa_65_keypair: MlDsa65Signer, pub mac_keys: Vec, pub storage_path: PathBuf, } @@ -285,6 +286,7 @@ impl NodePrivateConfig { keypair: dummy_signer(), bls_keypair: dummy_bls_signer(), ml_dsa_44_keypair: dummy_ml_dsa_44_signer(), + ml_dsa_65_keypair: dummy_ml_dsa_65_signer(), mac_keys: Vec::new(), storage_path: PathBuf::from("storage"), } @@ -293,16 +295,21 @@ impl NodePrivateConfig { pub fn new_for_benchmarks(working_dir: &Path, committee_size: usize) -> Vec { let signers = Signer::new_for_test(committee_size); let bls_signers = BlsSigner::new_for_test(committee_size); - let ml_dsa_signers = MlDsa44Signer::new_for_test(committee_size); + let ml_dsa_44_signers = MlDsa44Signer::new_for_test(committee_size); + let ml_dsa_65_signers = MlDsa65Signer::new_for_test(committee_size); let mac_keyrings = mac_keyrings_for_test(committee_size); signers .into_iter() .zip(bls_signers) - .zip(ml_dsa_signers) + .zip(ml_dsa_44_signers) + .zip(ml_dsa_65_signers) .zip(mac_keyrings) .enumerate() .map( - |(i, (((keypair, bls_keypair), ml_dsa_44_keypair), mac_keys))| { + |( + i, + ((((keypair, bls_keypair), ml_dsa_44_keypair), ml_dsa_65_keypair), mac_keys), + )| { let authority = i as AuthorityIndex; let path = working_dir.join(NodePrivateConfig::default_storage_path(authority)); Self { @@ -310,6 +317,7 @@ impl NodePrivateConfig { keypair, bls_keypair, ml_dsa_44_keypair, + ml_dsa_65_keypair, mac_keys, storage_path: path, } diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index 2f0d3eab..abfeeced 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -20,7 +20,9 @@ use crate::{ linearizer::CommittedSubDag, universal_committer::{UniversalCommitter, UniversalCommitterBuilder}, }, - crypto::{self, AsBytes, BlsSignatureBytes, BlsSigner, MacKey, MlDsa44Signer, Signer}, + crypto::{ + self, AsBytes, BlsSignatureBytes, BlsSigner, MacKey, MlDsa44Signer, MlDsa65Signer, Signer, + }, dag_state::{ ByzantineStrategy, CACHED_ROUNDS, CommitData, ConsensusProtocol, DagState, DataSource, OwnBlockData, @@ -62,6 +64,7 @@ pub struct Core { signer: Signer, bls_signer: BlsSigner, ml_dsa_44_signer: MlDsa44Signer, + ml_dsa_65_signer: MlDsa65Signer, mac_keys: Arc>, partial_sig_outbox: Option>, // todo - ugly, probably need to merge syncer and core @@ -189,6 +192,7 @@ impl Core { signer: private_config.keypair, bls_signer: private_config.bls_keypair, ml_dsa_44_signer: private_config.ml_dsa_44_keypair, + ml_dsa_65_signer: private_config.ml_dsa_65_keypair, mac_keys: Arc::new(private_config.mac_keys), partial_sig_outbox, recovered_committed_blocks: Some(committed_blocks), @@ -1004,6 +1008,7 @@ impl Core { BlockAuthenticationScheme::Ed25519 => BlockAuthorizer::Ed25519(&self.signer), BlockAuthenticationScheme::MacVector => BlockAuthorizer::MacVector(&self.mac_keys), BlockAuthenticationScheme::MlDsa44 => BlockAuthorizer::MlDsa44(&self.ml_dsa_44_signer), + BlockAuthenticationScheme::MlDsa65 => BlockAuthorizer::MlDsa65(&self.ml_dsa_65_signer), }; let mut block = VerifiedBlock::new_with_authorizer_and_unprovable( self.authority, diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 43ddaa25..99735e8f 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -6,7 +6,7 @@ use std::fmt; use blst::min_sig as bls; use ml_dsa::{ - Keypair as _, MlDsa44, Signature as MlDsaSignature, Signer as MlDsaSignerTrait, + Keypair as _, MlDsa44, MlDsa65, Signature as MlDsaSignature, Signer as MlDsaSignerTrait, SigningKey as MlDsaSigningKey, Verifier as MlDsaVerifierTrait, VerifyingKey as MlDsaVerifyingKey, }; @@ -79,9 +79,13 @@ pub const SIGNATURE_SIZE: usize = 64; pub const BLOCK_DIGEST_SIZE: usize = 32; pub const MAC_KEY_SIZE: usize = 32; pub const MAC_TAG_SIZE: usize = 32; -pub const ML_DSA_44_SEED_SIZE: usize = 32; +pub const ML_DSA_SEED_SIZE: usize = 32; +pub const ML_DSA_44_SEED_SIZE: usize = ML_DSA_SEED_SIZE; pub const ML_DSA_44_PUBLIC_KEY_SIZE: usize = 1_312; pub const ML_DSA_44_SIGNATURE_SIZE: usize = 2_420; +pub const ML_DSA_65_SEED_SIZE: usize = ML_DSA_SEED_SIZE; +pub const ML_DSA_65_PUBLIC_KEY_SIZE: usize = 1_952; +pub const ML_DSA_65_SIGNATURE_SIZE: usize = 3_309; pub const TRANSACTIONS_DIGEST_SIZE: usize = 32; @@ -104,16 +108,6 @@ pub struct MacKey([u8; MAC_KEY_SIZE]); #[derive(Clone, Copy, Ord, PartialOrd)] pub struct MacTag([u8; MAC_TAG_SIZE]); -#[derive(Clone, Eq, PartialEq)] -pub struct MlDsa44SignatureBytes(Box<[u8; ML_DSA_44_SIGNATURE_SIZE]>); - -#[derive(Clone)] -pub struct MlDsa44PublicKey(MlDsaVerifyingKey); - -/// Boxed so moving this wrapper does not copy private key material. -#[derive(Clone)] -pub struct MlDsa44Signer(Box>); - // Box ensures value is not copied in memory when Signer itself is moved around // for better security #[derive(Clone)] @@ -602,131 +596,199 @@ impl<'de> Deserialize<'de> for MacTag { } } -impl MlDsa44SignatureBytes { - pub fn from_bytes(bytes: [u8; ML_DSA_44_SIGNATURE_SIZE]) -> Self { - Self(Box::new(bytes)) - } -} - -impl AsRef<[u8]> for MlDsa44SignatureBytes { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } -} +macro_rules! define_ml_dsa_variant { + ( + parameter_set = $parameter_set:ty, + signature = $signature:ident, + public_key = $public_key:ident, + signer = $signer:ident, + seed_size = $seed_size:ident, + public_key_size = $public_key_size:ident, + signature_size = $signature_size:ident, + test_rng_seed = $test_rng_seed:expr, + label = $label:literal, + dummy_signer = $dummy_signer:ident, + dummy_public_key = $dummy_public_key:ident + ) => { + #[derive(Clone, Eq, PartialEq)] + pub struct $signature(Box<[u8; $signature_size]>); + + #[derive(Clone)] + pub struct $public_key(MlDsaVerifyingKey<$parameter_set>); + + /// Boxed so moving this wrapper does not copy private key material. + #[derive(Clone)] + pub struct $signer(Box>); + + impl $signature { + pub fn from_bytes(bytes: [u8; $signature_size]) -> Self { + Self(Box::new(bytes)) + } + } -impl fmt::Debug for MlDsa44SignatureBytes { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "MlDsa44Sig({})", &hex::encode(&self.0[..4])) - } -} + impl AsRef<[u8]> for $signature { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } + } -impl Serialize for MlDsa44SignatureBytes { - fn serialize(&self, serializer: S) -> Result { - serialize_fixed_bytes(self.0.as_ref(), serializer) - } -} + impl fmt::Debug for $signature { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}Sig({})", $label, &hex::encode(&self.0[..4])) + } + } -impl<'de> Deserialize<'de> for MlDsa44SignatureBytes { - fn deserialize>(deserializer: D) -> Result { - deserialize_fixed_bytes::(deserializer, "ML-DSA-44 signature") - .map(Self::from_bytes) - } -} + impl Serialize for $signature { + fn serialize(&self, serializer: S) -> Result { + serialize_fixed_bytes(self.0.as_ref(), serializer) + } + } -impl MlDsa44PublicKey { - pub fn from_bytes(bytes: &[u8; ML_DSA_44_PUBLIC_KEY_SIZE]) -> Self { - let encoded = ml_dsa::EncodedVerifyingKey::::from(*bytes); - Self(MlDsaVerifyingKey::decode(&encoded)) - } + impl<'de> Deserialize<'de> for $signature { + fn deserialize>(deserializer: D) -> Result { + deserialize_fixed_bytes::( + deserializer, + concat!($label, " signature"), + ) + .map(Self::from_bytes) + } + } - pub fn to_bytes(&self) -> [u8; ML_DSA_44_PUBLIC_KEY_SIZE] { - self.0.encode().into() - } + impl $public_key { + pub fn from_bytes(bytes: &[u8; $public_key_size]) -> Self { + let encoded = ml_dsa::EncodedVerifyingKey::<$parameter_set>::from(*bytes); + Self(MlDsaVerifyingKey::decode(&encoded)) + } + + pub fn to_bytes(&self) -> [u8; $public_key_size] { + self.0.encode().into() + } + + pub fn verify_digest_signature( + &self, + digest: &BlockDigest, + signature: &$signature, + ) -> Result<(), ml_dsa::signature::Error> { + let signature = MlDsaSignature::<$parameter_set>::try_from(signature.as_ref())?; + self.0.verify(digest.as_ref(), &signature) + } + } - pub fn verify_digest_signature( - &self, - digest: &BlockDigest, - signature: &MlDsa44SignatureBytes, - ) -> Result<(), ml_dsa::signature::Error> { - let signature = MlDsaSignature::::try_from(signature.as_ref())?; - self.0.verify(digest.as_ref(), &signature) - } -} + impl PartialEq for $public_key { + fn eq(&self, other: &Self) -> bool { + self.to_bytes() == other.to_bytes() + } + } -impl PartialEq for MlDsa44PublicKey { - fn eq(&self, other: &Self) -> bool { - self.to_bytes() == other.to_bytes() - } -} + impl Eq for $public_key {} -impl Eq for MlDsa44PublicKey {} + impl fmt::Debug for $public_key { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}Pk({})", $label, &hex::encode(&self.to_bytes()[..4])) + } + } -impl fmt::Debug for MlDsa44PublicKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "MlDsa44Pk({})", &hex::encode(&self.to_bytes()[..4])) - } -} + impl Serialize for $public_key { + fn serialize(&self, serializer: S) -> Result { + serialize_fixed_bytes(&self.to_bytes(), serializer) + } + } -impl Serialize for MlDsa44PublicKey { - fn serialize(&self, serializer: S) -> Result { - serialize_fixed_bytes(&self.to_bytes(), serializer) - } -} + impl<'de> Deserialize<'de> for $public_key { + fn deserialize>(deserializer: D) -> Result { + let bytes = deserialize_fixed_bytes::( + deserializer, + concat!($label, " public key"), + )?; + Ok(Self::from_bytes(&bytes)) + } + } -impl<'de> Deserialize<'de> for MlDsa44PublicKey { - fn deserialize>(deserializer: D) -> Result { - let bytes = deserialize_fixed_bytes::( - deserializer, - "ML-DSA-44 public key", - )?; - Ok(Self::from_bytes(&bytes)) - } -} + impl $signer { + pub fn new_for_test(n: usize) -> Vec { + let mut rng = StdRng::seed_from_u64($test_rng_seed); + (0..n) + .map(|_| { + let mut bytes = [0; $seed_size]; + rng.fill_bytes(&mut bytes); + let seed = ml_dsa::Seed::from(bytes); + Self(Box::new(MlDsaSigningKey::from_seed(&seed))) + }) + .collect() + } + + pub fn sign_digest(&self, digest: &BlockDigest) -> $signature { + let signature: MlDsaSignature<$parameter_set> = self.0.sign(digest.as_ref()); + $signature::from_bytes(signature.encode().into()) + } + + pub fn public_key(&self) -> $public_key { + $public_key(self.0.verifying_key()) + } + } -impl MlDsa44Signer { - pub fn new_for_test(n: usize) -> Vec { - let mut rng = StdRng::seed_from_u64(0x4d4c_4453_4134_3400); - (0..n) - .map(|_| { - let mut bytes = [0; ML_DSA_44_SEED_SIZE]; - rng.fill_bytes(&mut bytes); - let seed = ml_dsa::Seed::from(bytes); - Self(Box::new(MlDsaSigningKey::from_seed(&seed))) - }) - .collect() - } + impl fmt::Debug for $signer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}Signer(public_key={:?})", $label, self.public_key()) + } + } - pub fn sign_digest(&self, digest: &BlockDigest) -> MlDsa44SignatureBytes { - let signature: MlDsaSignature = self.0.sign(digest.as_ref()); - MlDsa44SignatureBytes::from_bytes(signature.encode().into()) - } + impl Serialize for $signer { + fn serialize(&self, serializer: S) -> Result { + let seed: [u8; $seed_size] = self.0.to_seed().into(); + serialize_fixed_bytes(&seed, serializer) + } + } - pub fn public_key(&self) -> MlDsa44PublicKey { - MlDsa44PublicKey(self.0.verifying_key()) - } -} + impl<'de> Deserialize<'de> for $signer { + fn deserialize>(deserializer: D) -> Result { + let bytes = deserialize_fixed_bytes::( + deserializer, + concat!($label, " seed"), + )?; + let seed = ml_dsa::Seed::from(bytes); + Ok(Self(Box::new(MlDsaSigningKey::from_seed(&seed)))) + } + } -impl fmt::Debug for MlDsa44Signer { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "MlDsa44Signer(public_key={:?})", self.public_key()) - } -} + pub fn $dummy_signer() -> $signer { + let seed = ml_dsa::Seed::from([0; $seed_size]); + $signer(Box::new(MlDsaSigningKey::from_seed(&seed))) + } -impl Serialize for MlDsa44Signer { - fn serialize(&self, serializer: S) -> Result { - let seed: [u8; ML_DSA_44_SEED_SIZE] = self.0.to_seed().into(); - serialize_fixed_bytes(&seed, serializer) - } + pub fn $dummy_public_key() -> $public_key { + $dummy_signer().public_key() + } + }; } -impl<'de> Deserialize<'de> for MlDsa44Signer { - fn deserialize>(deserializer: D) -> Result { - let bytes = - deserialize_fixed_bytes::(deserializer, "ML-DSA-44 seed")?; - let seed = ml_dsa::Seed::from(bytes); - Ok(Self(Box::new(MlDsaSigningKey::from_seed(&seed)))) - } -} +define_ml_dsa_variant!( + parameter_set = MlDsa44, + signature = MlDsa44SignatureBytes, + public_key = MlDsa44PublicKey, + signer = MlDsa44Signer, + seed_size = ML_DSA_44_SEED_SIZE, + public_key_size = ML_DSA_44_PUBLIC_KEY_SIZE, + signature_size = ML_DSA_44_SIGNATURE_SIZE, + test_rng_seed = 0x4d4c_4453_4134_3400, + label = "ML-DSA-44", + dummy_signer = dummy_ml_dsa_44_signer, + dummy_public_key = dummy_ml_dsa_44_public_key +); + +define_ml_dsa_variant!( + parameter_set = MlDsa65, + signature = MlDsa65SignatureBytes, + public_key = MlDsa65PublicKey, + signer = MlDsa65Signer, + seed_size = ML_DSA_65_SEED_SIZE, + public_key_size = ML_DSA_65_PUBLIC_KEY_SIZE, + signature_size = ML_DSA_65_SIGNATURE_SIZE, + test_rng_seed = 0x4d4c_4453_4136_3500, + label = "ML-DSA-65", + dummy_signer = dummy_ml_dsa_65_signer, + dummy_public_key = dummy_ml_dsa_65_public_key +); impl PublicKey { pub fn to_bytes(&self) -> [u8; 32] { @@ -923,15 +985,6 @@ pub fn dummy_public_key() -> PublicKey { dummy_signer().public_key() } -pub fn dummy_ml_dsa_44_signer() -> MlDsa44Signer { - let seed = ml_dsa::Seed::from([0; ML_DSA_44_SEED_SIZE]); - MlDsa44Signer(Box::new(MlDsaSigningKey::from_seed(&seed))) -} - -pub fn dummy_ml_dsa_44_public_key() -> MlDsa44PublicKey { - dummy_ml_dsa_44_signer().public_key() -} - // --------------------------------------------------------------------------- // BLS12-381 types (min_sig variant: 96-byte G2 public keys, 48-byte G1 // signatures). @@ -1278,6 +1331,33 @@ mod tests { assert_eq!(signature, decoded_signature); } + #[test] + fn ml_dsa_65_sign_verify_and_serde_roundtrip() { + let signer = MlDsa65Signer::new_for_test(1).pop().unwrap(); + let public_key = signer.public_key(); + let digest = BlockDigest([9; BLOCK_DIGEST_SIZE]); + let signature = signer.sign_digest(&digest); + + assert!( + public_key + .verify_digest_signature(&digest, &signature) + .is_ok() + ); + assert!( + public_key + .verify_digest_signature(&BlockDigest([8; BLOCK_DIGEST_SIZE]), &signature) + .is_err() + ); + + let encoded_key = bincode::serialize(&public_key).unwrap(); + let decoded_key: MlDsa65PublicKey = bincode::deserialize(&encoded_key).unwrap(); + let encoded_signature = bincode::serialize(&signature).unwrap(); + let decoded_signature: MlDsa65SignatureBytes = + bincode::deserialize(&encoded_signature).unwrap(); + assert_eq!(public_key, decoded_key); + assert_eq!(signature, decoded_signature); + } + #[test] fn bls_sign_verify_roundtrip() { let signers = BlsSigner::new_for_test(3); @@ -1342,6 +1422,9 @@ mod tests { ml_dsa_44_signer: MlDsa44Signer, ml_dsa_44_public_key: MlDsa44PublicKey, ml_dsa_44_signature: MlDsa44SignatureBytes, + ml_dsa_65_signer: MlDsa65Signer, + ml_dsa_65_public_key: MlDsa65PublicKey, + ml_dsa_65_signature: MlDsa65SignatureBytes, } #[test] @@ -1353,6 +1436,7 @@ mod tests { let block_digest = BlockDigest([7u8; BLOCK_DIGEST_SIZE]); let mac_tag = mac_key.compute_tag(0, 1, &block_digest); let ml_dsa_44_signer = dummy_ml_dsa_44_signer(); + let ml_dsa_65_signer = dummy_ml_dsa_65_signer(); let fixture = CryptoYamlFixture { signer, public_key, @@ -1367,6 +1451,9 @@ mod tests { ml_dsa_44_public_key: ml_dsa_44_signer.public_key(), ml_dsa_44_signature: ml_dsa_44_signer.sign_digest(&block_digest), ml_dsa_44_signer, + ml_dsa_65_public_key: ml_dsa_65_signer.public_key(), + ml_dsa_65_signature: ml_dsa_65_signer.sign_digest(&block_digest), + ml_dsa_65_signer, }; let yaml = serde_yaml::to_string(&fixture).unwrap(); @@ -1390,6 +1477,12 @@ mod tests { fixture.ml_dsa_44_signer.public_key(), decoded.ml_dsa_44_signer.public_key() ); + assert_eq!(fixture.ml_dsa_65_public_key, decoded.ml_dsa_65_public_key); + assert_eq!(fixture.ml_dsa_65_signature, decoded.ml_dsa_65_signature); + assert_eq!( + fixture.ml_dsa_65_signer.public_key(), + decoded.ml_dsa_65_signer.public_key() + ); assert_eq!( fixture.bls_signer.public_key(), decoded.bls_signer.public_key() diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index f724d892..2c282194 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -303,45 +303,30 @@ pub struct ProtocolConfig { impl ProtocolConfig { pub fn from_str(value: &str) -> Result { - let (consensus_protocol, block_authentication_scheme) = match value { - "starfish-mac" => ( - ConsensusProtocol::Starfish, - BlockAuthenticationScheme::MacVector, - ), - "starfish-ml-dsa-44" => ( - ConsensusProtocol::Starfish, - BlockAuthenticationScheme::MlDsa44, - ), - "starfish-speed-mac" => ( - ConsensusProtocol::StarfishSpeed, - BlockAuthenticationScheme::MacVector, - ), - "starfish-speed-ml-dsa-44" => ( - ConsensusProtocol::StarfishSpeed, - BlockAuthenticationScheme::MlDsa44, - ), - "sparse-starfish-speed-mac" => ( - ConsensusProtocol::SparseStarfishSpeed, - BlockAuthenticationScheme::MacVector, - ), - "sparse-starfish-speed-ml-dsa-44" => ( - ConsensusProtocol::SparseStarfishSpeed, - BlockAuthenticationScheme::MlDsa44, - ), - "bluestreak-mac" => ( - ConsensusProtocol::Bluestreak, - BlockAuthenticationScheme::MacVector, - ), - "bluestreak-ml-dsa-44" => ( - ConsensusProtocol::Bluestreak, - BlockAuthenticationScheme::MlDsa44, - ), - known => ( - ConsensusProtocol::from_known_str(known) - .ok_or_else(|| format!("Unknown consensus protocol '{known}'"))?, - BlockAuthenticationScheme::Ed25519, - ), - }; + let (protocol_name, block_authentication_scheme) = [ + ("-ml-dsa-65", BlockAuthenticationScheme::MlDsa65), + ("-ml-dsa-44", BlockAuthenticationScheme::MlDsa44), + ("-mac", BlockAuthenticationScheme::MacVector), + ] + .into_iter() + .find_map(|(suffix, scheme)| value.strip_suffix(suffix).map(|base| (base, scheme))) + .unwrap_or((value, BlockAuthenticationScheme::Ed25519)); + + let consensus_protocol = ConsensusProtocol::from_known_str(protocol_name) + .ok_or_else(|| format!("Unknown consensus protocol '{value}'"))?; + if block_authentication_scheme != BlockAuthenticationScheme::Ed25519 + && !matches!( + consensus_protocol, + ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishSpeed + | ConsensusProtocol::SparseStarfishSpeed + | ConsensusProtocol::Bluestreak + ) + { + return Err(format!( + "Block authentication variants are not supported for '{protocol_name}'" + )); + } Ok(Self { consensus_protocol, block_authentication_scheme, @@ -4969,6 +4954,13 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("starfish-ml-dsa-65").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Starfish, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, + } + ); assert_eq!( ProtocolConfig::from_str("starfish-speed").unwrap(), ProtocolConfig { @@ -4990,6 +4982,13 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("starfish-speed-ml-dsa-65").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::StarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, + } + ); assert_eq!( ProtocolConfig::from_str("sparse-starfish-speed").unwrap(), ProtocolConfig { @@ -5011,6 +5010,13 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("sparse-starfish-speed-ml-dsa-65").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, + } + ); assert_eq!( ProtocolConfig::from_str("bluestreak").unwrap(), ProtocolConfig { @@ -5032,6 +5038,14 @@ mod tests { block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, } ); + assert_eq!( + ProtocolConfig::from_str("bluestreak-ml-dsa-65").unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::Bluestreak, + block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, + } + ); + assert!(ProtocolConfig::from_str("mysticeti-ml-dsa-65").is_err()); assert!(ProtocolConfig::from_str("starfish-unknown").is_err()); assert!(ProtocolConfig::from_str("starfish-speed-unknown").is_err()); assert!(ProtocolConfig::from_str("sparse-starfish-speed-unknown").is_err()); diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 72ce9e46..39773db1 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -42,7 +42,8 @@ use crate::{ crypto, crypto::{ AsBytes, BlsSignatureBytes, BlsSigner, CryptoHash, MacKey, MacTag, MlDsa44SignatureBytes, - MlDsa44Signer, SignatureBytes, Signer, TransactionsCommitment, + MlDsa44Signer, MlDsa65SignatureBytes, MlDsa65Signer, SignatureBytes, Signer, + TransactionsCommitment, }, dag_state::ConsensusProtocol, data::{Data, IN_MEMORY_BLOCKS, IN_MEMORY_BLOCKS_BYTES}, @@ -263,6 +264,7 @@ pub enum BlockAuthentication { /// Recipient-specific authenticator selected from a full vector by a relay. MacTag(MacTag), MlDsa44(MlDsa44SignatureBytes), + MlDsa65(MlDsa65SignatureBytes), } mod flat_mac_vector { @@ -333,12 +335,14 @@ pub enum BlockAuthenticationScheme { Ed25519, MacVector, MlDsa44, + MlDsa65, } pub enum BlockAuthorizer<'a> { Ed25519(&'a Signer), MacVector(&'a [MacKey]), MlDsa44(&'a MlDsa44Signer), + MlDsa65(&'a MlDsa65Signer), } impl BlockAuthorizer<'_> { @@ -362,6 +366,9 @@ impl BlockAuthorizer<'_> { Self::MlDsa44(signer) => { BlockAuthentication::MlDsa44(signer.sign_digest(content_digest)) } + Self::MlDsa65(signer) => { + BlockAuthentication::MlDsa65(signer.sign_digest(content_digest)) + } } } } @@ -1609,6 +1616,14 @@ impl VerifiedBlock { bail!("Block ML-DSA-44 verification has failed: {error:?}"); } } + (BlockAuthenticationScheme::MlDsa65, BlockAuthentication::MlDsa65(signature)) => { + let Some(public_key) = committee.get_ml_dsa_65_public_key(self.authority()) else { + bail!("Unknown block author {}", self.authority()) + }; + if let Err(error) = public_key.verify_digest_signature(&digest, signature) { + bail!("Block ML-DSA-65 verification has failed: {error:?}"); + } + } (expected, actual) => { bail!("Expected {expected:?} block authentication, received {actual:?}") } @@ -2475,27 +2490,33 @@ mod tests { fn block_reference_depends_only_on_content_across_authentication_schemes() { let committee = Committee::new_for_benchmarks(4); let ed_signers = Signer::new_for_test(committee.len()); - let ml_dsa_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); + let ml_dsa_44_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); + let ml_dsa_65_signers = crypto::MlDsa65Signer::new_for_test(committee.len()); let mac_keyrings = crypto::mac_keyrings_for_test(committee.len()); let ed = BlockAuthorizer::Ed25519(&ed_signers[0]); let mac = BlockAuthorizer::MacVector(&mac_keyrings[0]); - let ml_dsa = BlockAuthorizer::MlDsa44(&ml_dsa_signers[0]); + let ml_dsa_44 = BlockAuthorizer::MlDsa44(&ml_dsa_44_signers[0]); + let ml_dsa_65 = BlockAuthorizer::MlDsa65(&ml_dsa_65_signers[0]); let ed_block = make_authenticated_starfish_block(&committee, &ed); let mac_block = make_authenticated_starfish_block(&committee, &mac); - let ml_dsa_block = make_authenticated_starfish_block(&committee, &ml_dsa); + let ml_dsa_44_block = make_authenticated_starfish_block(&committee, &ml_dsa_44); + let ml_dsa_65_block = make_authenticated_starfish_block(&committee, &ml_dsa_65); assert_eq!(ed_block.reference(), mac_block.reference()); - assert_eq!(ed_block.reference(), ml_dsa_block.reference()); + assert_eq!(ed_block.reference(), ml_dsa_44_block.reference()); + assert_eq!(ed_block.reference(), ml_dsa_65_block.reference()); assert_ne!(ed_block.authentication(), mac_block.authentication()); - assert_ne!(ed_block.authentication(), ml_dsa_block.authentication()); + assert_ne!(ed_block.authentication(), ml_dsa_44_block.authentication()); + assert_ne!(ed_block.authentication(), ml_dsa_65_block.authentication()); } #[test] fn all_authentication_schemes_verify_for_starfish_protocols() { let committee = Committee::new_for_benchmarks(4); let ed_signers = Signer::new_for_test(committee.len()); - let ml_dsa_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); + let ml_dsa_44_signers = crypto::MlDsa44Signer::new_for_test(committee.len()); + let ml_dsa_65_signers = crypto::MlDsa65Signer::new_for_test(committee.len()); let mac_keyrings = crypto::mac_keyrings_for_test(committee.len()); for consensus_protocol in [ @@ -2529,10 +2550,18 @@ mod tests { make_authenticated_starfish_block_for_author( &committee, author as AuthorityIndex, - &BlockAuthorizer::MlDsa44(&ml_dsa_signers[author]), + &BlockAuthorizer::MlDsa44(&ml_dsa_44_signers[author]), ), BlockAuthenticationScheme::MlDsa44, ), + ( + make_authenticated_starfish_block_for_author( + &committee, + author as AuthorityIndex, + &BlockAuthorizer::MlDsa65(&ml_dsa_65_signers[author]), + ), + BlockAuthenticationScheme::MlDsa65, + ), ]; for (block, scheme) in &cases { for (receiver, receiver_keys) in mac_keyrings.iter().enumerate() { diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index d2c213ab..f5b06f4b 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -73,6 +73,15 @@ impl Validator { )); } } + BlockAuthenticationScheme::MlDsa65 => { + if committee.get_ml_dsa_65_public_key(authority) + != Some(&private_config.ml_dsa_65_keypair.public_key()) + { + return Err(eyre!( + "ML-DSA-65 private key does not match committee authority {authority}" + )); + } + } } // Network and metrics setup remains the same let network_address = public_config @@ -343,9 +352,11 @@ mod smoke_tests { #[test_case("starfish", 60)] #[test_case("starfish-mac", 700)] #[test_case("starfish-ml-dsa-44", 720)] + #[test_case("starfish-ml-dsa-65", 1000)] #[test_case("starfish-speed", 80)] #[test_case("starfish-speed-mac", 760)] #[test_case("starfish-speed-ml-dsa-44", 780)] + #[test_case("starfish-speed-ml-dsa-65", 1040)] #[test_case("starfish-bls", 100)] #[test_case("sailfish++", 120)] #[test_case("bluestreak", 140)] @@ -353,8 +364,10 @@ mod smoke_tests { #[test_case("sparse-starfish-speed", 180)] #[test_case("sparse-starfish-speed-mac", 840)] #[test_case("sparse-starfish-speed-ml-dsa-44", 860)] + #[test_case("sparse-starfish-speed-ml-dsa-65", 1080)] #[test_case("bluestreak-mac", 920)] #[test_case("bluestreak-ml-dsa-44", 940)] + #[test_case("bluestreak-ml-dsa-65", 1120)] #[tokio::test] async fn validator_commit(consensus: &str, port_offset: u16) { run_commit_test(consensus, port_offset).await; @@ -450,9 +463,12 @@ mod smoke_tests { #[test_case("cordial-miners", 140)] #[test_case("starfish", 160)] #[test_case("starfish-mac", 740)] + #[test_case("starfish-ml-dsa-44", 1020)] + #[test_case("starfish-ml-dsa-65", 1200)] #[test_case("starfish-speed", 180)] #[test_case("starfish-speed-mac", 800)] #[test_case("starfish-speed-ml-dsa-44", 820)] + #[test_case("starfish-speed-ml-dsa-65", 1220)] #[test_case("starfish-bls", 200)] #[test_case("sailfish++", 220)] #[test_case("bluestreak", 260)] @@ -460,8 +476,10 @@ mod smoke_tests { #[test_case("sparse-starfish-speed", 320)] #[test_case("sparse-starfish-speed-mac", 880)] #[test_case("sparse-starfish-speed-ml-dsa-44", 900)] + #[test_case("sparse-starfish-speed-ml-dsa-65", 1240)] #[test_case("bluestreak-mac", 960)] #[test_case("bluestreak-ml-dsa-44", 980)] + #[test_case("bluestreak-ml-dsa-65", 1260)] #[tokio::test] async fn validator_sync(consensus: &str, port_offset: u16) { run_sync_test(consensus, port_offset).await; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 12b3cef9..11a5f588 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -67,7 +67,7 @@ enum Operation { #[clap(long, value_name = "STRING", default_value = "")] byzantine_strategy: String, /// Consensus/authentication variant (for example `starfish-mac`, - /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-44`). + /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-65`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, }, @@ -97,7 +97,7 @@ enum Operation { #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, /// Consensus/authentication variant (for example `starfish-mac`, - /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-44`). + /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-65`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, /// Directory to store validator data (default: current directory) @@ -149,7 +149,7 @@ enum Operation { #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, /// Consensus/authentication variant (for example `starfish-mac`, - /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-44`). + /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-65`). #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, #[clap(long, value_name = "INT", default_value_t = 600)] diff --git a/local-dryrun/README.md b/local-dryrun/README.md index f2c54f69..8b4e0b7b 100644 --- a/local-dryrun/README.md +++ b/local-dryrun/README.md @@ -32,13 +32,15 @@ NUM_NODES=10 DESIRED_TPS=1000 CONSENSUS=starfish \ | `CONSENSUS` | `bluestreak` | Consensus protocol (see below) | | `TEST_TIME` | `3000` | Experiment duration in seconds | -Supported `CONSENSUS` values include the Ed25519, MAC, and ML-DSA-44 variants +Supported `CONSENSUS` values include the Ed25519, MAC, ML-DSA-44, and ML-DSA-65 variants of Starfish, Starfish Speed, Sparse-Starfish-Speed, and Bluestreak: -`starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-speed`, -`starfish-speed-mac`, `starfish-speed-ml-dsa-44`, +`starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-ml-dsa-65`, +`starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, +`starfish-speed-ml-dsa-65`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, -`sparse-starfish-speed-ml-dsa-44`, `bluestreak`, `bluestreak-mac`, -`bluestreak-ml-dsa-44`, `starfish-bls`, `cordial-miners`, `mysticeti`, +`sparse-starfish-speed-ml-dsa-44`, `sparse-starfish-speed-ml-dsa-65`, +`bluestreak`, `bluestreak-mac`, `bluestreak-ml-dsa-44`, +`bluestreak-ml-dsa-65`, `starfish-bls`, `cordial-miners`, `mysticeti`, `sailfish-pp`, and `mysticeti-bls`. ### Protocol Tuning diff --git a/local-dryrun/dryrun.sh b/local-dryrun/dryrun.sh index 37982a4c..117404d1 100755 --- a/local-dryrun/dryrun.sh +++ b/local-dryrun/dryrun.sh @@ -6,7 +6,7 @@ NUM_NODES=${NUM_NODES:-10} NUM_CRASHED_NODES=${NUM_CRASHED_NODES:-0} DESIRED_TPS=${DESIRED_TPS:-100} -# Authentication variants append -mac or -ml-dsa-44 to starfish, +# Authentication variants append -mac, -ml-dsa-44, or -ml-dsa-65 to starfish, # starfish-speed, sparse-starfish-speed, or bluestreak. # Other options: starfish-bls, cordial-miners, mysticeti, # sailfish-pp, mysticeti-bls From 707bfae424ff2bdba994bcec89adc73f61630a26 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:04:51 +0200 Subject: [PATCH 16/62] Record ML-DSA-65 authentication benchmarks --- README.md | 4 +- ...sh-authentication-geo-40-validators-60s.md | 46 +++++++++---- ...14-authentication-geo-60-validators-60s.md | 65 +++++++++++++------ 3 files changed, 81 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 8d844c37..bab9266b 100644 --- a/README.md +++ b/README.md @@ -96,8 +96,8 @@ treated as production-ready cryptography. See the [40-validator geographic authentication comparison](benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md) -for matching Ed25519, MAC-vector, and ML-DSA-44 measurements across all four -protocol families. A separate +for matching Ed25519, MAC-vector, ML-DSA-44, and ML-DSA-65 measurements across +all four protocol families. A separate [60-validator moderate-load comparison](benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md) records the single-machine scaling boundary at 600 tx/s aggregate load. diff --git a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md index 697eb443..e52471f3 100644 --- a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md +++ b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md @@ -1,7 +1,8 @@ # Starfish and Bluestreak authentication comparison — 40-validator geographic emulation Date: 2026-07-13–14
-Source revisions: `c90f8fb` (Starfish families), `3562890` (Bluestreak)
+Source revisions: `c90f8fb` (original Starfish families), `3562890` +(original Bluestreak), `3a82c04` (ML-DSA-65 extension)
Host: Apple Silicon (`arm64`), macOS 15.7.4
Build: Rust 1.86.0, release profile @@ -45,17 +46,21 @@ target/release/starfish local-benchmark \ | Starfish | Ed25519 | 590.92 | 644.60 | 790.63 | 407.02 | 5.28 | 5.27 | 13.67 | | Starfish | MAC vector | 592.98 | 646.42 | 792.77 | 393.48 | 5.63 | 5.63 | 14.54 | | Starfish | ML-DSA-44 | 571.38 | 627.02 | 792.33 | 399.95 | 7.44 | 7.44 | 19.22 | +| Starfish | ML-DSA-65 | 567.80 | 624.50 | 794.25 | 406.15 | 7.81 | 7.81 | 20.15 | | Starfish Speed | Ed25519 | 712.02 | 843.95 | 794.25 | 338.55 | 4.39 | 4.39 | 11.32 | | Starfish Speed | MAC vector | 679.67 | 788.67 | 794.10 | 344.63 | 4.68 | 4.67 | 12.06 | | Starfish Speed | ML-DSA-44 | 648.98 | 733.05 | 794.35 | 354.83 | 6.54 | 6.53 | 16.85 | +| Starfish Speed | ML-DSA-65 | 520.70 | 601.35 | 795.93 | 367.73 | 7.61 | 7.60 | 19.57 | | Sparse-Starfish-Speed | Ed25519 | 446.98 | 507.93 | 795.08 | 367.70 | 0.85 | 0.85 | 2.19 | | Sparse-Starfish-Speed | MAC vector | 441.60 | 502.15 | 790.58 | 366.55 | 1.23 | 1.23 | 3.19 | | Sparse-Starfish-Speed | ML-DSA-44 | 435.23 | 499.48 | 794.33 | 371.28 | 2.38 | 2.38 | 6.14 | +| Sparse-Starfish-Speed | ML-DSA-65 | 434.75 | 495.95 | 788.70 | 377.72 | 2.92 | 2.92 | 7.59 | | Bluestreak | Ed25519 | 417.65 | 477.93 | 793.48 | 389.53 | 0.59 | 0.59 | 1.53 | | Bluestreak | MAC vector | 418.95 | 478.07 | 793.38 | 363.75 | 0.99 | 0.99 | 2.56 | | Bluestreak | ML-DSA-44 | 420.70 | 478.43 | 795.43 | 368.57 | 1.41 | 1.41 | 3.62 | +| Bluestreak | ML-DSA-65 | 421.02 | 479.50 | 796.03 | 364.83 | 1.71 | 1.70 | 4.39 | -All twelve commands exited successfully after printing their metrics. No +All sixteen commands exited successfully after printing their metrics. No deserialize, socket-buffer, or transport errors were observed during these runs. @@ -65,12 +70,16 @@ runs. |---|---|---:|---:|---:|---:|---:| | Starfish | MAC vector | +0.3% | +0.3% | +0.3% | -3.3% | +6.6% | | Starfish | ML-DSA-44 | -3.3% | -2.7% | +0.2% | -1.7% | +40.9% | +| Starfish | ML-DSA-65 | -3.9% | -3.1% | +0.5% | -0.2% | +47.9% | | Starfish Speed | MAC vector | -4.5% | -6.6% | 0.0% | +1.8% | +6.6% | | Starfish Speed | ML-DSA-44 | -8.9% | -13.1% | 0.0% | +4.8% | +49.0% | +| Starfish Speed | ML-DSA-65 | -26.9% | -28.7% | +0.2% | +8.6% | +73.3% | | Sparse-Starfish-Speed | MAC vector | -1.2% | -1.1% | -0.6% | -0.3% | +44.7% | | Sparse-Starfish-Speed | ML-DSA-44 | -2.6% | -1.7% | -0.1% | +1.0% | +180.0% | +| Sparse-Starfish-Speed | ML-DSA-65 | -2.7% | -2.4% | -0.8% | +2.7% | +243.5% | | Bluestreak | MAC vector | +0.3% | 0.0% | 0.0% | -6.6% | +67.8% | | Bluestreak | ML-DSA-44 | +0.7% | +0.1% | +0.2% | -5.4% | +139.0% | +| Bluestreak | ML-DSA-65 | +0.8% | +0.3% | +0.3% | -6.3% | +189.8% | The lower latency values in some MAC and ML-DSA runs must not be interpreted as an authentication speedup. These are single, sequential trials without a @@ -83,46 +92,54 @@ randomized order or variance estimates. | Starfish Speed | Ed25519 | +20.5% | +30.9% | +0.5% | -16.8% | -16.9% | | Starfish Speed | MAC vector | +14.6% | +22.0% | +0.2% | -12.4% | -16.9% | | Starfish Speed | ML-DSA-44 | +13.6% | +16.9% | +0.3% | -11.3% | -12.1% | +| Starfish Speed | ML-DSA-65 | -8.3% | -3.7% | +0.2% | -9.5% | -2.6% | | Sparse-Starfish-Speed | Ed25519 | -24.4% | -21.2% | +0.6% | -9.7% | -83.9% | | Sparse-Starfish-Speed | MAC vector | -25.5% | -22.3% | -0.3% | -6.8% | -78.2% | | Sparse-Starfish-Speed | ML-DSA-44 | -23.8% | -20.3% | +0.3% | -7.2% | -68.0% | +| Sparse-Starfish-Speed | ML-DSA-65 | -23.4% | -20.6% | -0.7% | -7.0% | -62.6% | | Bluestreak | Ed25519 | -29.3% | -25.9% | +0.4% | -4.3% | -88.8% | | Bluestreak | MAC vector | -29.3% | -26.0% | +0.1% | -7.6% | -82.4% | | Bluestreak | ML-DSA-44 | -26.4% | -23.7% | +0.4% | -7.8% | -81.0% | +| Bluestreak | ML-DSA-65 | -25.9% | -23.2% | +0.2% | -10.2% | -78.1% | ## Interpretation - The corrected workload is 1,000 tx/s total, not 4,000 tx/s. The harness divides the total evenly, so every validator generates 25 tx/s. - Authentication choice did not materially affect committed throughput. All - variants reported 790.58-795.43 TPS, a spread of 0.6% across the complete + variants reported 788.70-796.03 TPS, a spread of 0.9% across the complete matrix. - The local harness includes the 12-second connection warm-up in its 60-second TPS denominator: the default 10 seconds plus 2 seconds for a 40-validator committee. Roughly 48 seconds therefore submit transactions; - the measured 790.58-795.43 TPS corresponds to about 988-994 tx/s during the + the measured 788.70-796.03 TPS corresponds to about 986-995 tx/s during the active submission window, close to the offered 1,000 tx/s. - For plain Starfish and Starfish Speed, MAC raises outbound bandwidth by - 6.6% over Ed25519. ML-DSA-44 raises it by 40.9% and 49.0%, respectively. + 6.6% over Ed25519. ML-DSA-44 raises it by 40.9% and 49.0%, while ML-DSA-65 + raises it by 47.9% and 73.3%, respectively. - Bluestreak's baseline protocol traffic is even leaner. Its MAC vector raises outbound bandwidth from 0.59 to 0.99 MB/s (+67.8%), while ML-DSA-44 raises - it to 1.41 MB/s (+139.0%). Latency and throughput remain within 0.7% of its - Ed25519 baseline. + it to 1.41 MB/s (+139.0%) and ML-DSA-65 to 1.71 MB/s (+189.8%). Latency and + throughput remain within 0.8% of its Ed25519 baseline. - Sparse-Starfish-Speed removes so much baseline protocol traffic that authentication bytes become a larger fraction of the remainder. Its MAC variant rises from 0.85 to 1.23 MB/s (+44.7%), and ML-DSA-44 rises to - 2.38 MB/s (+180.0%). The absolute traffic remains below every matching - non-sparse variant. + 2.38 MB/s (+180.0%); ML-DSA-65 reaches 2.92 MB/s (+243.5%). The absolute + traffic remains below every matching non-sparse variant. +- The signature sizes explain the incremental ML-DSA bandwidth: Ed25519 is + 64 bytes, ML-DSA-44 is 2,420 bytes, and ML-DSA-65 is 3,309 bytes. ML-DSA-65 + public keys are 1,952 bytes but are provisioned in the committee config, + not sent in every block. - The MAC result is consistent with the current design: direct author block streaming carries the full committee-sized MAC vector, while relay and synchronization paths carry one recipient tag. Consequently, the remaining author-stream authentication cost grows with committee size. - Bluestreak is the strongest 40-validator result in this local emulation: - 418-421 ms block latency, about 478 ms end-to-end latency, and - 0.59-1.41 MB/s outbound across the three authentication schemes. Relative + 418-421 ms block latency, 478-480 ms end-to-end latency, and + 0.59-1.71 MB/s outbound across the four authentication schemes. Relative to matching Sparse-Starfish-Speed authentication, it reduces block latency - by 3.3-6.6%, end-to-end latency by 4.2-5.9%, and outbound bandwidth by - 19.5-40.8%. + by 3.2-6.6%, end-to-end latency by 3.3-5.9%, and outbound bandwidth by + 19.5-41.4%. - Starfish Speed alone was slower than plain Starfish at 40 validators even though it was faster in the earlier 10-validator experiment. This reversal points to a single-host scaling or run-variance effect and needs randomized, @@ -135,6 +152,9 @@ randomized order or variance estimates. clearest result; latency differences need repeated trials. - The Bluestreak runs were made one day after the Starfish-family runs using the same host and benchmark configuration but a newer source revision. +- The ML-DSA-65 rows were appended after the original matrix on source + revision `3a82c04`; they use the same host, release profile, and command + template. - Four validators share each synthetic region profile. This produces the AWS delay distribution but does not model independent machines or real WAN bandwidth constraints. diff --git a/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md b/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md index 3182c901..bdc746ae 100644 --- a/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md +++ b/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md @@ -1,7 +1,8 @@ # Authentication comparison — 60-validator geographic emulation Date: 2026-07-14
-Source revision: `4553e5e`
+Source revisions: `4553e5e` (original matrix), `3a82c04` (ML-DSA-65 +extension)
Host: Apple Silicon (`arm64`), macOS 15.7.4
Build: Rust 1.86.0, release profile @@ -42,18 +43,27 @@ target/release/starfish local-benchmark \ | Starfish | Ed25519 | 664.45 | 728.77 | 444.30 | 515.02 | 8.59 | 8.58 | 39.59 | | Starfish | MAC vector | 680.68 | 753.42 | 414.87 | 449.23 | 8.35 | 8.34 | 41.21 | | Starfish | ML-DSA-44 | 849.38 | 937.02 | 427.77 | 417.95 | 9.72 | 9.71 | 46.53 | +| Starfish | ML-DSA-65† | 749.38 | 837.48 | 413.53 | 443.48 | 13.98 | 13.98 | 69.25 | | Starfish Speed | Ed25519 | 592.13 | 673.87 | 460.37 | 467.12 | 6.63 | 6.62 | 29.49 | | Starfish Speed | MAC vector | 670.80 | 783.93 | 430.95 | 439.30 | 7.30 | 7.30 | 34.70 | | Starfish Speed | ML-DSA-44 | 641.70 | 726.03 | 433.63 | 461.57 | 11.06 | 11.05 | 52.22 | +| Starfish Speed | ML-DSA-65 | 641.40 | 727.28 | 421.98 | 415.88 | 12.10 | 12.09 | 58.71 | | Sparse-Starfish-Speed | Ed25519 | 467.70 | 525.98 | 466.22 | 535.57 | 0.88 | 0.88 | 3.87 | | Sparse-Starfish-Speed | MAC vector | 467.67 | 528.23 | 466.62 | 551.88 | 1.84 | 1.84 | 8.08 | | Sparse-Starfish-Speed | ML-DSA-44 | 464.87 | 526.97 | 467.07 | 528.88 | 2.98 | 2.98 | 13.08 | +| Sparse-Starfish-Speed | ML-DSA-65 | 461.05 | 516.02 | 469.62 | 560.13 | 3.95 | 3.95 | 17.24 | | Bluestreak | Ed25519 | 422.53 | 480.58 | 468.22 | 551.08 | 0.52 | 0.52 | 2.28 | | Bluestreak | MAC vector | 421.62 | 480.37 | 467.63 | 545.72 | 1.47 | 1.46 | 6.42 | | Bluestreak | ML-DSA-44 | 422.72 | 481.70 | 467.88 | 547.13 | 1.75 | 1.75 | 7.67 | +| Bluestreak | ML-DSA-65 | 423.98 | 482.82 | 470.25 | 548.02 | 2.22 | 2.22 | 9.66 | -All twelve commands exited successfully after emitting their metrics. No -authentication, deserialize, socket-buffer, or transport errors were observed. +All sixteen commands exited successfully after emitting their metrics. The +Starfish ML-DSA-65 run marked † crossed this host's socket-buffer ceiling at +second 20: three writes reported `No buffer space available`, followed by +three decompress/deserialize warnings. Its metrics are retained as a +host-contended stress result, not a clean protocol comparison. The other +three ML-DSA-65 runs had no authentication, deserialize, socket-buffer, or +transport errors. ## Warm-up-adjusted offered-rate utilization @@ -67,15 +77,19 @@ seconds. The following values estimate the active rate as `TPS × 60 / 47`. | Starfish | Ed25519 | 567.19 | 94.5% | | Starfish | MAC vector | 529.62 | 88.3% | | Starfish | ML-DSA-44 | 546.09 | 91.0% | +| Starfish | ML-DSA-65† | 527.91 | 88.0% | | Starfish Speed | Ed25519 | 587.71 | 98.0% | | Starfish Speed | MAC vector | 550.15 | 91.7% | | Starfish Speed | ML-DSA-44 | 553.57 | 92.3% | +| Starfish Speed | ML-DSA-65 | 538.70 | 89.8% | | Sparse-Starfish-Speed | Ed25519 | 595.17 | 99.2% | | Sparse-Starfish-Speed | MAC vector | 595.69 | 99.3% | | Sparse-Starfish-Speed | ML-DSA-44 | 596.26 | 99.4% | +| Sparse-Starfish-Speed | ML-DSA-65 | 599.51 | 99.9% | | Bluestreak | Ed25519 | 597.73 | 99.6% | | Bluestreak | MAC vector | 596.97 | 99.5% | | Bluestreak | ML-DSA-44 | 597.29 | 99.5% | +| Bluestreak | ML-DSA-65 | 600.32 | 100.1% | ## Relative to Ed25519 within each protocol @@ -83,41 +97,50 @@ seconds. The following values estimate the active rate as `TPS × 60 / 47`. |---|---|---:|---:|---:|---:|---:| | Starfish | MAC vector | +2.4% | +3.4% | -6.6% | -12.8% | -2.8% | | Starfish | ML-DSA-44 | +27.8% | +28.6% | -3.7% | -18.8% | +13.2% | +| Starfish | ML-DSA-65† | +12.8% | +14.9% | -6.9% | -13.9% | +62.7% | | Starfish Speed | MAC vector | +13.3% | +16.3% | -6.4% | -6.0% | +10.1% | | Starfish Speed | ML-DSA-44 | +8.4% | +7.7% | -5.8% | -1.2% | +66.8% | +| Starfish Speed | ML-DSA-65 | +8.3% | +7.9% | -8.3% | -11.0% | +82.5% | | Sparse-Starfish-Speed | MAC vector | 0.0% | +0.4% | +0.1% | +3.0% | +109.1% | | Sparse-Starfish-Speed | ML-DSA-44 | -0.6% | +0.2% | +0.2% | -1.2% | +238.6% | +| Sparse-Starfish-Speed | ML-DSA-65 | -1.4% | -1.9% | +0.7% | +4.6% | +348.9% | | Bluestreak | MAC vector | -0.2% | 0.0% | -0.1% | -1.0% | +182.7% | | Bluestreak | ML-DSA-44 | 0.0% | +0.2% | -0.1% | -0.7% | +236.5% | +| Bluestreak | ML-DSA-65 | +0.3% | +0.5% | +0.4% | -0.6% | +326.9% | Raw bandwidth can fall despite a larger authentication proof when a run produces fewer blocks, as in plain Starfish MAC. The bandwidth-efficiency metric—the ratio of bytes sent to committed transaction-payload bytes—rises -from 39.59 to 41.21 and captures the normalized increase. +from 39.59 to 41.21 for MAC and 69.25 for ML-DSA-65, capturing the normalized +increase even when raw block production varies. ## Interpretation - Sixty validators are feasible on this machine at a 600 tx/s aggregate offered load, but headroom depends strongly on the protocol family. -- Sparse-Starfish-Speed and Bluestreak sustain 99.2-99.6% of the active offered - rate for every authentication scheme. Their latency and throughput vary by - at most 0.6% and 0.2%, respectively, within each family. -- Plain Starfish sustains 88.3-94.5% of the active offered rate, while - Starfish Speed sustains 91.7-98.0%. Their authentication comparisons are - therefore partly measurements of shared-host contention. In particular, - plain Starfish ML-DSA-44 has 27.8% higher block latency than its Ed25519 run. +- Sparse-Starfish-Speed and Bluestreak sustain 99.2-100.1% of the active + offered rate across all four authentication schemes. Within each family, + block latency varies by at most 1.4% and throughput by at most 0.7%. +- Plain Starfish sustains 88.0-94.5% of the active offered rate, while + Starfish Speed sustains 89.8-98.0%. Their authentication comparisons are + therefore partly measurements of shared-host contention. Plain Starfish + ML-DSA-65 is the clearest limit: its 13.98 MB/s full-mesh traffic triggered + the host's socket-buffer errors, so that row is not a clean protocol result. - Bluestreak has the lowest latency and absolute bandwidth across every - authentication scheme: 422-423 ms block latency, 480-482 ms end-to-end - latency, and 0.52-1.75 MB/s outbound. + authentication scheme: 422-424 ms block latency, 480-483 ms end-to-end + latency, and 0.52-2.22 MB/s outbound. - Relative to matching Sparse-Starfish-Speed authentication, Bluestreak lowers - block latency by 9.1-9.8%, end-to-end latency by 8.6-9.1%, and outbound - bandwidth by 20.1-41.3%, with nearly identical throughput. + block latency by 8.0-9.8%, end-to-end latency by 6.4-9.1%, and outbound + bandwidth by 20.1-43.8%, with nearly identical throughput. - At 60 validators, a full author MAC vector contains 60 × 32 = 1,920 bytes, compared with a 64-byte Ed25519 signature and a 2,420-byte ML-DSA-44 - signature. Relays and synchronization responses still carry only one - 32-byte recipient tag. Because Bluestreak and Sparse-Starfish-Speed remove - most other traffic, these authentication bytes produce large percentages - while their absolute bandwidth remains well below the denser protocols. + signature; an ML-DSA-65 signature is 3,309 bytes. ML-DSA-65 public keys are + 1,952 bytes and are provisioned in committee configuration rather than sent + in each block. Relays and synchronization responses for MAC blocks still + carry only one 32-byte recipient tag. Because Bluestreak and + Sparse-Starfish-Speed remove most other traffic, authentication bytes + produce large percentages while absolute bandwidth remains well below the + denser protocols. - For repeatable all-protocol authentication comparisons on this single host, 40 validators remains the safer configuration. Sixty validators is a useful stress configuration and a clean operating point for the sparse and @@ -128,6 +151,10 @@ from 39.59 to 41.21 and captures the normalized increase. - There is one sequential run per configuration, with no randomized order or confidence interval. Host scheduling and thermal state can affect results. +- The ML-DSA-65 rows were appended after the original matrix on source + revision `3a82c04`. The marked plain-Starfish row encountered host transport + errors and should be treated only as evidence that this configuration + exceeds the local machine's clean operating point. - The 600 tx/s load differs from the earlier 40-validator 1,000 tx/s matrix; the two experiments should not be treated as a pure committee-size scaling comparison. From acd271bb9d5cb26f0992d306c0a9a3163d6d0b65 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:24:07 +0200 Subject: [PATCH 17/62] Remove unrelated changes from authentication PR --- ...026-07-13-starfish-authentication-local.md | 13 +--- .../2026-07-13-validator-scaling-probe.md | 63 ------------------- crates/starfish-core/src/block_handler.rs | 6 +- .../src/bls_certificate_aggregator.rs | 21 ++++--- crates/starfish-core/src/encoder.rs | 3 +- .../src/transactions_generator.rs | 3 +- crates/starfish-core/src/types.rs | 7 +-- crates/starfish/src/main.rs | 33 +++++----- 8 files changed, 38 insertions(+), 111 deletions(-) delete mode 100644 benchmark-results/2026-07-13-validator-scaling-probe.md diff --git a/benchmark-results/2026-07-13-starfish-authentication-local.md b/benchmark-results/2026-07-13-starfish-authentication-local.md index e51ad802..b4d05f11 100644 --- a/benchmark-results/2026-07-13-starfish-authentication-local.md +++ b/benchmark-results/2026-07-13-starfish-authentication-local.md @@ -1,8 +1,8 @@ # Starfish authentication comparison — local Apple Silicon -Date: 2026-07-13 +Date: 2026-07-13
Source revision: `8a3bded` plus the Sparse authentication changes committed with this report
-Host: Apple Silicon (`arm64`), macOS 15.7.4 +Host: Apple Silicon (`arm64`), macOS 15.7.4
Build: Rust 1.86.0, release profile ## Configuration @@ -95,14 +95,7 @@ target/release/starfish local-benchmark \ storage, and network stack. These results are useful for directional local comparison, not distributed capacity claims. -## Benchmark harness fix - -The previous local benchmark shutdown aborted validator tasks and immediately -deleted their RocksDB directories. On macOS this left benchmark parents stuck -in an uninterruptible exiting state. The harness now uses a `JoinSet`, aborts -all validator tasks, drains them completely, and only then removes storage. -The validation run and all nine measured protocol/authentication combinations -exited normally. +## Benchmark validation note During the first Sparse MAC run, the receiver-side transport guard exposed a round-gap response that still carried full MAC vectors. The sender now routes diff --git a/benchmark-results/2026-07-13-validator-scaling-probe.md b/benchmark-results/2026-07-13-validator-scaling-probe.md deleted file mode 100644 index 0f722eb4..00000000 --- a/benchmark-results/2026-07-13-validator-scaling-probe.md +++ /dev/null @@ -1,63 +0,0 @@ -# Local validator-count scaling probe - -Date: 2026-07-13
-Source revision: `dee67ee`
-Host: Apple Silicon (`arm64`), macOS 15.7.4, 64 GiB RAM, 16 logical CPU cores - -## Purpose - -Find a practical committee-size limit for geographic latency emulation on one -machine. Offered load stays at 100 tx/s per validator. The 20-80-validator -probes run for 30 seconds under the AWS RTT latency model. The 10-validator -row is the earlier 60-second baseline and is included only for orientation. - -## Sparse-Starfish-Speed MAC results - -| Validators | Offered load | Duration | Block latency (ms) | E2E latency (ms) | TPS | BPS | Outbound (MB/s) | Outcome | -|---:|---:|---:|---:|---:|---:|---:|---:|---| -| 10 | 1,000 | 60 s | 418.7 | 483.5 | 816.25 | 92.70 | 0.47 | Clean baseline | -| 20 | 2,000 | 30 s | 435.85 | 501.05 | 1,240.83 | 191.67 | 0.88 | Clean | -| 40 | 4,000 | 30 s | 461.45 | 545.75 | 2,336.33 | 364.13 | 1.96 | Clean | -| 64 | 6,400 | 30 s | 480.42 | 540.67 | 3,435.23 | 591.83 | 3.47 | Clean, near resource saturation | -| 80 | 8,000 | 30 s | 662.36 | 752.25 | 2,238.30 | 470.77 | 2.86 | Unhealthy: socket-buffer and decode errors | - -During the active 64-validator run, the process reached about 1,032% CPU -(roughly 10 cores) and 17.7 GiB resident memory. At 80 validators the network -failed to establish and sustain the full mesh reliably: the run emitted -deserialization warnings and repeated macOS `No buffer space available` -errors. Its throughput regression and latency jump therefore mark it as an -invalid benchmark configuration on this host. - -## Common-size check with plain Starfish MAC - -| Validators | Offered load | Duration | Block latency (ms) | E2E latency (ms) | TPS | BPS | Outbound (MB/s) | Cancelled reconstructions | -|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| 40 | 4,000 | 30 s | 596.08 | 650.00 | 2,331.33 | 366.10 | 5.71 | 184 | - -Plain Starfish completed cleanly at 40 validators but used about 2.9x the -outbound bandwidth of Sparse-Starfish-Speed MAC at the same committee size. -It therefore provides the more conservative limit for a matrix that compares -all protocol families. - -## Limits and recommendation - -- The hard code limit is 512 validators (`MAX_COMMITTEE_SIZE`). This is a type - and data-structure bound, not a realistic single-machine target. -- The process file-descriptor limit is 1,048,575, and benchmark ports remain - well within `u16` even at 512 validators. Neither is the first constraint. -- The local network is a full mesh, so peer relationships grow as `n(n-1)`: - 1,560 at 40 validators, 4,032 at 64, and 6,320 at 80. Each validator also - owns a RocksDB instance. Socket buffers, connection tasks, and database - memory dominate before the hard committee or descriptor limits. -- Use 20 validators for quick, low-risk development comparisons. -- Use 40 validators as the recommended maximum for repeatable comparisons - across Starfish, Starfish Speed, Sparse, and all authentication schemes. -- Treat 64 as a Sparse-only local stress configuration, not a comfortable - full-matrix setting. -- Use multiple machines through the orchestrator beyond 40 validators. For - committees above ten, the local AWS table repeats the same ten regions while - all validators still share one kernel and physical host. - -The 30-second probes have different warm-up proportions from the 60-second -baseline, so their TPS values should not be used as a formal scaling curve. -The clean/error boundary and sampled resource usage are the relevant signals. diff --git a/crates/starfish-core/src/block_handler.rs b/crates/starfish-core/src/block_handler.rs index 871a381a..001fa0a0 100644 --- a/crates/starfish-core/src/block_handler.rs +++ b/crates/starfish-core/src/block_handler.rs @@ -33,9 +33,8 @@ const REAL_BLOCK_HANDLER_TXN_GEN_STEP: usize = 32; const _: () = assert_constants(); #[allow(dead_code)] -#[allow(clippy::manual_is_multiple_of)] const fn assert_constants() { - if REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0 { + if !REAL_BLOCK_HANDLER_TXN_SIZE.is_multiple_of(REAL_BLOCK_HANDLER_TXN_GEN_STEP) { panic!("REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0") } } @@ -265,7 +264,6 @@ impl RealCommitHandler { } impl CommitObserver for RealCommitHandler { - #[allow(clippy::manual_is_multiple_of)] fn handle_commit( &mut self, dag_state: &DagState, @@ -294,7 +292,7 @@ impl CommitObserver for RealCommitHandler { let digest_short = u16::from_le_bytes([self.commit_digest[0], self.commit_digest[1]]) & 0x3FF; self.metrics.commit_digest_latest.set(digest_short as i64); - if commit_index % 100 == 0 { + if commit_index.is_multiple_of(100) { self.metrics.commit_digest.set(digest_short as i64); } diff --git a/crates/starfish-core/src/bls_certificate_aggregator.rs b/crates/starfish-core/src/bls_certificate_aggregator.rs index 9f9fe847..865dedcd 100644 --- a/crates/starfish-core/src/bls_certificate_aggregator.rs +++ b/crates/starfish-core/src/bls_certificate_aggregator.rs @@ -297,15 +297,18 @@ impl BlsCertificateAggregator { } else if let Some(&origin_index) = seen_leader_certs.get(&(*leader_ref, *cert)) { push_task_source(&mut origins[origin_index], source); - } else if let Some(task) = self - .aggregate_same_message_task(crypto::bls_leader_message(leader_ref), cert) - { - tasks.push(BlsVerificationTask { - block_index: origins.len(), - ..task - }); - seen_leader_certs.insert((*leader_ref, *cert), origins.len()); - origins.push(TaskOrigin::AggLeader(*leader_ref, *cert, vec![source])); + } else { + if let Some(task) = self.aggregate_same_message_task( + crypto::bls_leader_message(leader_ref), + cert, + ) { + tasks.push(BlsVerificationTask { + block_index: origins.len(), + ..task + }); + seen_leader_certs.insert((*leader_ref, *cert), origins.len()); + origins.push(TaskOrigin::AggLeader(*leader_ref, *cert, vec![source])); + } } } } diff --git a/crates/starfish-core/src/encoder.rs b/crates/starfish-core/src/encoder.rs index a1e0bc01..122b2e48 100644 --- a/crates/starfish-core/src/encoder.rs +++ b/crates/starfish-core/src/encoder.rs @@ -42,7 +42,6 @@ impl ShardEncoder for Encoder { data } - #[allow(clippy::manual_is_multiple_of)] fn encode_transactions( &mut self, block: &[BaseTransaction], @@ -58,7 +57,7 @@ impl ShardEncoder for Encoder { let mut shard_bytes = (bytes_length + 4).div_ceil(info_length); // Ensure shard_bytes meets alignment requirements (must be multiple of 2). - if shard_bytes % 2 != 0 { + if !shard_bytes.is_multiple_of(2) { shard_bytes += 1; } diff --git a/crates/starfish-core/src/transactions_generator.rs b/crates/starfish-core/src/transactions_generator.rs index 3725fe6d..0ca474f9 100644 --- a/crates/starfish-core/src/transactions_generator.rs +++ b/crates/starfish-core/src/transactions_generator.rs @@ -59,7 +59,6 @@ impl TransactionGenerator { ); } - #[allow(clippy::manual_is_multiple_of)] pub async fn run(mut self) { let load = self.parameters.load; let max_transactions_per_block_interval = load.div_ceil(Self::BATCHES_IN_SECOND); @@ -183,7 +182,7 @@ impl TransactionGenerator { return; } - if counter % 10_000 == 0 { + if counter.is_multiple_of(10_000) { self.metrics .submitted_transactions_bytes .inc_by(tx_to_report * tx_size as u64); diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 39773db1..df6a36ec 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -304,9 +304,9 @@ mod flat_mac_vector { ) } - #[allow(clippy::manual_is_multiple_of)] fn visit_bytes(self, bytes: &[u8]) -> Result { - if bytes.len() % crypto::MAC_TAG_SIZE != 0 { + let chunks = bytes.chunks_exact(crypto::MAC_TAG_SIZE); + if !chunks.remainder().is_empty() { return Err(E::custom(format!( "invalid flat MAC vector length {}; expected a multiple of {}", bytes.len(), @@ -314,8 +314,7 @@ mod flat_mac_vector { ))); } - Ok(bytes - .chunks_exact(crypto::MAC_TAG_SIZE) + Ok(chunks .map(|chunk| { let mut tag = [0; crypto::MAC_TAG_SIZE]; tag.copy_from_slice(chunk); diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 11a5f588..ee02e1c5 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -28,7 +28,6 @@ use starfish_core::{ types::AuthorityIndex, validator::Validator, }; -use tokio::task::JoinSet; use tokio::time::Instant; use tracing_subscriber::{EnvFilter, filter::LevelFilter, fmt}; @@ -329,7 +328,6 @@ fn benchmark_genesis( Ok(()) } -#[allow(clippy::manual_is_multiple_of)] async fn local_benchmark( committee_size: usize, mut load: usize, @@ -387,7 +385,8 @@ async fn local_benchmark( let base_dir = PathBuf::from("local-benchmark"); fs::create_dir_all(&base_dir)?; - let mut validator_tasks = JoinSet::new(); + let mut handles = Vec::with_capacity(committee_size); + let mut abort_handles = Vec::with_capacity(committee_size); let mut metrics_of_honest_validators = Vec::new(); let mut reporters_of_honest_validators = Vec::new(); @@ -429,7 +428,7 @@ async fn local_benchmark( )); } } - let is_byzantine = authority % 3 == 0 && authority / 3 < num_byzantine_nodes; + let is_byzantine = authority.is_multiple_of(3) && authority / 3 < num_byzantine_nodes; let validator = if is_byzantine { Validator::start( authority as AuthorityIndex, @@ -459,10 +458,12 @@ async fn local_benchmark( } // Use the same pattern as the run method - validator_tasks.spawn(async move { + let handle = tokio::spawn(async move { let (network_result, _metrics_result) = validator.await_completion().await; network_result }); + abort_handles.push(handle.abort_handle()); + handles.push(handle); } // Run for specified duration @@ -479,25 +480,23 @@ async fn local_benchmark( duration_secs, ); - // Abort and fully drain validator tasks before deleting their - // RocksDB directories. On macOS, removing storage while aborted - // tasks are still dropping database handles can leave the - // benchmark parent stuck in an uninterruptible exit state. - validator_tasks.abort_all(); - while validator_tasks.join_next().await.is_some() { + // Abort all tasks + for abort_handle in abort_handles { + abort_handle.abort(); } // Clean up fs::remove_dir_all(base_dir)?; Ok(()) } - result = validator_tasks.join_next() => { - running.store(false, Ordering::SeqCst); - tracing::warn!("Validator terminated before benchmark timeout: {result:?}"); - validator_tasks.abort_all(); - while validator_tasks.join_next().await.is_some() { + _ = async { + for handle in handles { + if let Err(e) = handle.await { + tracing::warn!("Validator terminated with error: {}", e); + } } - println!("A validator completed before timeout"); + } => { + println!("All validators completed before timeout"); Metrics::aggregate_and_display( metrics_of_honest_validators, reporters_of_honest_validators, From c5b019876d98bdd39476742c064415909dbb3922 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:48:44 +0200 Subject: [PATCH 18/62] Separate signature selection from MAC experiments --- README.md | 71 ++--- ...sh-authentication-geo-40-validators-60s.md | 164 ------------ ...6-07-13-starfish-authentication-geo-60s.md | 102 ------- ...026-07-13-starfish-authentication-local.md | 104 -------- ...14-authentication-geo-60-validators-60s.md | 165 ------------ crates/orchestrator/README.md | 8 +- crates/orchestrator/src/benchmark.rs | 11 +- crates/orchestrator/src/main.rs | 30 ++- crates/starfish-core/src/config.rs | 5 + crates/starfish-core/src/dag_state.rs | 252 +++++++++--------- crates/starfish-core/src/net_sync.rs | 4 +- crates/starfish-core/src/validator.rs | 127 +++++---- crates/starfish/src/main.rs | 78 +++++- local-dryrun/README.md | 18 +- local-dryrun/dryrun.sh | 11 +- 15 files changed, 354 insertions(+), 796 deletions(-) delete mode 100644 benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md delete mode 100644 benchmark-results/2026-07-13-starfish-authentication-geo-60s.md delete mode 100644 benchmark-results/2026-07-13-starfish-authentication-local.md delete mode 100644 benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md diff --git a/README.md b/README.md index bab9266b..5b2db961 100644 --- a/README.md +++ b/README.md @@ -59,32 +59,29 @@ achieving 2-round optimistic commit latency. leader, data availability) in block headers, with async verification offloaded from the critical path. -### Block authentication experiments - -Starfish, Starfish Speed, Sparse-Starfish-Speed, and Bluestreak can each be run -with four interchangeable block-authentication schemes: - -| Protocol | Ed25519 | MAC vector | ML-DSA-44 | ML-DSA-65 | -|---|---|---|---|---| -| Starfish | `starfish` | `starfish-mac` | `starfish-ml-dsa-44` | `starfish-ml-dsa-65` | -| Starfish Speed | `starfish-speed` | `starfish-speed-mac` | `starfish-speed-ml-dsa-44` | `starfish-speed-ml-dsa-65` | -| Sparse-Starfish-Speed | `sparse-starfish-speed` | `sparse-starfish-speed-mac` | `sparse-starfish-speed-ml-dsa-44` | `sparse-starfish-speed-ml-dsa-65` | -| Bluestreak | `bluestreak` | `bluestreak-mac` | `bluestreak-ml-dsa-44` | `bluestreak-ml-dsa-65` | - -For all sixteen variants, `BlockReference.digest` is the BLAKE3 hash of the -canonical block content only. The authentication proof is a separate header -field and does not change the block reference. An author using a MAC variant -sends the full vector, with exactly one tag for every committee member, -to its direct recipients. A direct recipient retains that vector and, when -relaying a block or header, or answering a synchronization request, sends only -the destination's tag. A tag-only copy cannot be relayed a second time. -Receivers accept a full vector only through proactive block streaming directly -from the block's claimed author; relay and synchronization traffic must contain -exactly one recipient tag. If the same node later receives the author's -directly streamed full-vector copy, it upgrades the stored authentication -without adding a second DAG vertex and can then relay recipient-specific tags. -Benchmark genesis deterministically generates the pairwise MAC keys, ML-DSA -seeds, and public keys in the node configuration. +### Block authentication + +Every consensus protocol can select its block signature independently: + +| Scheme | CLI option | +|---|---| +| Ed25519 (default) | `--block-authentication ed25519` or omit the option | +| ML-DSA-44 | `--block-authentication ml-dsa-44` | +| ML-DSA-65 | `--block-authentication ml-dsa-65` | + +For example, `--consensus mysticeti --block-authentication ml-dsa-65` changes +Mysticeti's block signature without creating another consensus protocol. This +selection is also available through the orchestrator. Protocol-specific BLS +certificates are unaffected. These digital-signature selections retain the +transferable public verification assumed by the protocols and do not change +their message flow or proof structure. The same value can be set as +`block_authentication` in the node-parameters YAML; the CLI option overrides +that setting. + +`BlockReference.digest` is the BLAKE3 hash of the canonical block content only. +The modular authentication proof is a separate header field and does not change +the block reference. Benchmark genesis generates all Ed25519 and ML-DSA key +material regardless of the selected signature scheme. The ML-DSA wrappers are generated from a common parameter-set definition. ML-DSA-44 uses 1,312-byte public keys and 2,420-byte signatures; ML-DSA-65 @@ -94,12 +91,22 @@ This is research/benchmark code. The RustCrypto `ml-dsa` implementation used here states that it has not been independently audited and should not be treated as production-ready cryptography. -See the -[40-validator geographic authentication comparison](benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md) -for matching Ed25519, MAC-vector, ML-DSA-44, and ML-DSA-65 measurements across -all four protocol families. A separate -[60-validator moderate-load comparison](benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md) -records the single-machine scaling boundary at 600 tx/s aggregate load. +#### Experimental MAC protocols + +`starfish-mac`, `starfish-speed-mac`, `sparse-starfish-speed-mac`, and +`bluestreak-mac` remain separate work-in-progress benchmark protocols. They are +not interchangeable signature selections and cannot be combined with +`--block-authentication`. + +These variants measure a lower bound for pairwise-MAC authentication. Direct +author streaming carries the full committee-sized MAC vector; relays and +synchronization responses carry only the destination's tag. Pairwise MACs do +not provide transferable authorship, and a Byzantine author can give different +recipients valid and invalid tags for the same block reference. The current +prototype does not add the quorum-authentication/RBC exchange needed to bind +the author to an available authenticator. It therefore makes no safety or +liveness claim and must not be treated as a proven variant of the underlying +protocol. ## Dissemination Modes diff --git a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md b/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md deleted file mode 100644 index e52471f3..00000000 --- a/benchmark-results/2026-07-13-starfish-authentication-geo-40-validators-60s.md +++ /dev/null @@ -1,164 +0,0 @@ -# Starfish and Bluestreak authentication comparison — 40-validator geographic emulation - -Date: 2026-07-13–14
-Source revisions: `c90f8fb` (original Starfish families), `3562890` -(original Bluestreak), `3a82c04` (ML-DSA-65 extension)
-Host: Apple Silicon (`arm64`), macOS 15.7.4
-Build: Rust 1.86.0, release profile - -## Configuration - -- 40 honest validators in one local process -- 1,000 tx/s aggregate offered load (25 tx/s per validator) -- 60-second measurement window -- Protocol-default dissemination: `push-useful` for the Starfish families and - `pull` for Bluestreak -- One run per configuration -- Geographic latency emulation enabled; no uniform-latency override - -The latency harness has ten AWS region profiles. At 40 validators, validator -indices are mapped modulo ten, producing four validators per modeled region. -The RTT values are divided by two to obtain one-way delays and independent -±3% per-message jitter is applied. Base one-way delays range from 0.5 ms to -154.5 ms. - -This is a single-machine latency emulation, not a deployment on 40 remote -hosts. All validators share the host's CPU, memory, storage, loopback network, -and kernel socket resources. - -In this report, **Sparse-Starfish-Speed means the sparse implementation of -Starfish Speed; it is not Bluestreak.** - -Command template: - -```text -target/release/starfish local-benchmark \ - --committee-size 40 \ - --load 1000 \ - --consensus \ - --duration-secs 60 -``` - -## Results - -| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | -|---|---|---:|---:|---:|---:|---:|---:|---:| -| Starfish | Ed25519 | 590.92 | 644.60 | 790.63 | 407.02 | 5.28 | 5.27 | 13.67 | -| Starfish | MAC vector | 592.98 | 646.42 | 792.77 | 393.48 | 5.63 | 5.63 | 14.54 | -| Starfish | ML-DSA-44 | 571.38 | 627.02 | 792.33 | 399.95 | 7.44 | 7.44 | 19.22 | -| Starfish | ML-DSA-65 | 567.80 | 624.50 | 794.25 | 406.15 | 7.81 | 7.81 | 20.15 | -| Starfish Speed | Ed25519 | 712.02 | 843.95 | 794.25 | 338.55 | 4.39 | 4.39 | 11.32 | -| Starfish Speed | MAC vector | 679.67 | 788.67 | 794.10 | 344.63 | 4.68 | 4.67 | 12.06 | -| Starfish Speed | ML-DSA-44 | 648.98 | 733.05 | 794.35 | 354.83 | 6.54 | 6.53 | 16.85 | -| Starfish Speed | ML-DSA-65 | 520.70 | 601.35 | 795.93 | 367.73 | 7.61 | 7.60 | 19.57 | -| Sparse-Starfish-Speed | Ed25519 | 446.98 | 507.93 | 795.08 | 367.70 | 0.85 | 0.85 | 2.19 | -| Sparse-Starfish-Speed | MAC vector | 441.60 | 502.15 | 790.58 | 366.55 | 1.23 | 1.23 | 3.19 | -| Sparse-Starfish-Speed | ML-DSA-44 | 435.23 | 499.48 | 794.33 | 371.28 | 2.38 | 2.38 | 6.14 | -| Sparse-Starfish-Speed | ML-DSA-65 | 434.75 | 495.95 | 788.70 | 377.72 | 2.92 | 2.92 | 7.59 | -| Bluestreak | Ed25519 | 417.65 | 477.93 | 793.48 | 389.53 | 0.59 | 0.59 | 1.53 | -| Bluestreak | MAC vector | 418.95 | 478.07 | 793.38 | 363.75 | 0.99 | 0.99 | 2.56 | -| Bluestreak | ML-DSA-44 | 420.70 | 478.43 | 795.43 | 368.57 | 1.41 | 1.41 | 3.62 | -| Bluestreak | ML-DSA-65 | 421.02 | 479.50 | 796.03 | 364.83 | 1.71 | 1.70 | 4.39 | - -All sixteen commands exited successfully after printing their metrics. No -deserialize, socket-buffer, or transport errors were observed during these -runs. - -## Relative to Ed25519 within each protocol - -| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---|---:|---:|---:|---:|---:| -| Starfish | MAC vector | +0.3% | +0.3% | +0.3% | -3.3% | +6.6% | -| Starfish | ML-DSA-44 | -3.3% | -2.7% | +0.2% | -1.7% | +40.9% | -| Starfish | ML-DSA-65 | -3.9% | -3.1% | +0.5% | -0.2% | +47.9% | -| Starfish Speed | MAC vector | -4.5% | -6.6% | 0.0% | +1.8% | +6.6% | -| Starfish Speed | ML-DSA-44 | -8.9% | -13.1% | 0.0% | +4.8% | +49.0% | -| Starfish Speed | ML-DSA-65 | -26.9% | -28.7% | +0.2% | +8.6% | +73.3% | -| Sparse-Starfish-Speed | MAC vector | -1.2% | -1.1% | -0.6% | -0.3% | +44.7% | -| Sparse-Starfish-Speed | ML-DSA-44 | -2.6% | -1.7% | -0.1% | +1.0% | +180.0% | -| Sparse-Starfish-Speed | ML-DSA-65 | -2.7% | -2.4% | -0.8% | +2.7% | +243.5% | -| Bluestreak | MAC vector | +0.3% | 0.0% | 0.0% | -6.6% | +67.8% | -| Bluestreak | ML-DSA-44 | +0.7% | +0.1% | +0.2% | -5.4% | +139.0% | -| Bluestreak | ML-DSA-65 | +0.8% | +0.3% | +0.3% | -6.3% | +189.8% | - -The lower latency values in some MAC and ML-DSA runs must not be interpreted -as an authentication speedup. These are single, sequential trials without a -randomized order or variance estimates. - -## Protocol relative to matching Starfish authentication - -| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---|---:|---:|---:|---:|---:| -| Starfish Speed | Ed25519 | +20.5% | +30.9% | +0.5% | -16.8% | -16.9% | -| Starfish Speed | MAC vector | +14.6% | +22.0% | +0.2% | -12.4% | -16.9% | -| Starfish Speed | ML-DSA-44 | +13.6% | +16.9% | +0.3% | -11.3% | -12.1% | -| Starfish Speed | ML-DSA-65 | -8.3% | -3.7% | +0.2% | -9.5% | -2.6% | -| Sparse-Starfish-Speed | Ed25519 | -24.4% | -21.2% | +0.6% | -9.7% | -83.9% | -| Sparse-Starfish-Speed | MAC vector | -25.5% | -22.3% | -0.3% | -6.8% | -78.2% | -| Sparse-Starfish-Speed | ML-DSA-44 | -23.8% | -20.3% | +0.3% | -7.2% | -68.0% | -| Sparse-Starfish-Speed | ML-DSA-65 | -23.4% | -20.6% | -0.7% | -7.0% | -62.6% | -| Bluestreak | Ed25519 | -29.3% | -25.9% | +0.4% | -4.3% | -88.8% | -| Bluestreak | MAC vector | -29.3% | -26.0% | +0.1% | -7.6% | -82.4% | -| Bluestreak | ML-DSA-44 | -26.4% | -23.7% | +0.4% | -7.8% | -81.0% | -| Bluestreak | ML-DSA-65 | -25.9% | -23.2% | +0.2% | -10.2% | -78.1% | - -## Interpretation - -- The corrected workload is 1,000 tx/s total, not 4,000 tx/s. The harness - divides the total evenly, so every validator generates 25 tx/s. -- Authentication choice did not materially affect committed throughput. All - variants reported 788.70-796.03 TPS, a spread of 0.9% across the complete - matrix. -- The local harness includes the 12-second connection warm-up in its - 60-second TPS denominator: the default 10 seconds plus 2 seconds for a - 40-validator committee. Roughly 48 seconds therefore submit transactions; - the measured 788.70-796.03 TPS corresponds to about 986-995 tx/s during the - active submission window, close to the offered 1,000 tx/s. -- For plain Starfish and Starfish Speed, MAC raises outbound bandwidth by - 6.6% over Ed25519. ML-DSA-44 raises it by 40.9% and 49.0%, while ML-DSA-65 - raises it by 47.9% and 73.3%, respectively. -- Bluestreak's baseline protocol traffic is even leaner. Its MAC vector raises - outbound bandwidth from 0.59 to 0.99 MB/s (+67.8%), while ML-DSA-44 raises - it to 1.41 MB/s (+139.0%) and ML-DSA-65 to 1.71 MB/s (+189.8%). Latency and - throughput remain within 0.8% of its Ed25519 baseline. -- Sparse-Starfish-Speed removes so much baseline protocol traffic that - authentication bytes become a larger fraction of the remainder. Its MAC - variant rises from 0.85 to 1.23 MB/s (+44.7%), and ML-DSA-44 rises to - 2.38 MB/s (+180.0%); ML-DSA-65 reaches 2.92 MB/s (+243.5%). The absolute - traffic remains below every matching non-sparse variant. -- The signature sizes explain the incremental ML-DSA bandwidth: Ed25519 is - 64 bytes, ML-DSA-44 is 2,420 bytes, and ML-DSA-65 is 3,309 bytes. ML-DSA-65 - public keys are 1,952 bytes but are provisioned in the committee config, - not sent in every block. -- The MAC result is consistent with the current design: direct author block - streaming carries the full committee-sized MAC vector, while relay and - synchronization paths carry one recipient tag. Consequently, the remaining - author-stream authentication cost grows with committee size. -- Bluestreak is the strongest 40-validator result in this local emulation: - 418-421 ms block latency, 478-480 ms end-to-end latency, and - 0.59-1.71 MB/s outbound across the four authentication schemes. Relative - to matching Sparse-Starfish-Speed authentication, it reduces block latency - by 3.2-6.6%, end-to-end latency by 3.3-5.9%, and outbound bandwidth by - 19.5-41.4%. -- Starfish Speed alone was slower than plain Starfish at 40 validators even - though it was faster in the earlier 10-validator experiment. This reversal - points to a single-host scaling or run-variance effect and needs randomized, - repeated trials before it is treated as a protocol conclusion. - -## Caveats - -- There is one run per configuration and no randomized run order, warm-up - exclusion, or confidence interval. Relative authentication bandwidth is the - clearest result; latency differences need repeated trials. -- The Bluestreak runs were made one day after the Starfish-family runs using - the same host and benchmark configuration but a newer source revision. -- The ML-DSA-65 rows were appended after the original matrix on source - revision `3a82c04`; they use the same host, release profile, and command - template. -- Four validators share each synthetic region profile. This produces the AWS - delay distribution but does not model independent machines or real WAN - bandwidth constraints. -- The 40 validators form a full mesh of 1,560 directed peer relationships and - use one RocksDB instance each. Host contention is part of the measurement. -- The progress window starts before transaction generation, which explains - why displayed TPS is below the aggregate offered rate. diff --git a/benchmark-results/2026-07-13-starfish-authentication-geo-60s.md b/benchmark-results/2026-07-13-starfish-authentication-geo-60s.md deleted file mode 100644 index a662738f..00000000 --- a/benchmark-results/2026-07-13-starfish-authentication-geo-60s.md +++ /dev/null @@ -1,102 +0,0 @@ -# Starfish authentication comparison — 60-second geographic emulation - -Date: 2026-07-13
-Source revision: `d3f57c2`
-Host: Apple Silicon (`arm64`), macOS 15.7.4
-Build: Rust 1.86.0, release profile - -## Configuration - -- 10 honest validators in one local process -- 1,000 tx/s offered load (100 tx/s per validator) -- 60-second measurement window -- Default `push-useful` dissemination for all three protocols -- One run per configuration -- Geographic latency emulation enabled; no uniform-latency override - -The ten validators map in order to `us-east-1`, `us-west-1`, -`ca-central-1`, `eu-west-1`, `eu-south-1`, `eu-north-1`, `sa-east-1`, -`ap-south-1`, `ap-southeast-1`, and `ap-northeast-1`. The harness converts -its AWS RTT table to one-way delay by dividing each cell by two, then applies -independent ±3% per-message jitter. The resulting base one-way delays range -from 0.5 ms within a region to 154.5 ms between the most distant pair. - -This is a single-machine latency emulation, not a deployment on ten remote -hosts. CPU, storage, and the physical network stack remain shared. - -Command template: - -```text -target/release/starfish local-benchmark \ - --committee-size 10 \ - --load 1000 \ - --consensus \ - --duration-secs 60 -``` - -## Results - -| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | -|---|---|---:|---:|---:|---:|---:|---:|---:| -| Starfish | Ed25519 | 553.4 | 603.7 | 816.25 | 100.05 | 0.49 | 0.49 | 1.23 | -| Starfish | MAC vector | 559.7 | 612.1 | 815.75 | 94.93 | 0.53 | 0.53 | 1.34 | -| Starfish | ML-DSA-44 | 554.1 | 604.2 | 814.70 | 97.00 | 0.76 | 0.76 | 1.92 | -| Starfish Speed | Ed25519 | 460.6 | 519.7 | 814.83 | 94.33 | 0.51 | 0.51 | 1.29 | -| Starfish Speed | MAC vector | 458.6 | 522.6 | 816.08 | 95.18 | 0.54 | 0.54 | 1.35 | -| Starfish Speed | ML-DSA-44 | 457.7 | 518.5 | 817.83 | 94.13 | 0.77 | 0.77 | 1.93 | -| Sparse-Starfish-Speed | Ed25519 | 417.6 | 485.1 | 818.25 | 95.42 | 0.45 | 0.45 | 1.13 | -| Sparse-Starfish-Speed | MAC vector | 418.7 | 483.5 | 816.25 | 92.70 | 0.47 | 0.47 | 1.18 | -| Sparse-Starfish-Speed | ML-DSA-44 | 420.3 | 486.9 | 817.95 | 92.85 | 0.71 | 0.71 | 1.78 | - -## Relative to Ed25519 within each protocol - -| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---|---:|---:|---:|---:|---:| -| Starfish | MAC vector | +1.1% | +1.4% | -0.1% | -5.1% | +8.2% | -| Starfish | ML-DSA-44 | +0.1% | +0.1% | -0.2% | -3.0% | +55.1% | -| Starfish Speed | MAC vector | -0.4% | +0.6% | +0.2% | +0.9% | +5.9% | -| Starfish Speed | ML-DSA-44 | -0.6% | -0.2% | +0.4% | -0.2% | +51.0% | -| Sparse-Starfish-Speed | MAC vector | +0.3% | -0.3% | -0.2% | -2.9% | +4.4% | -| Sparse-Starfish-Speed | ML-DSA-44 | +0.6% | +0.4% | 0.0% | -2.7% | +57.8% | - -## Protocol relative to matching Starfish authentication - -| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---|---:|---:|---:|---:|---:| -| Starfish Speed | Ed25519 | -16.8% | -13.9% | -0.2% | -5.7% | +4.1% | -| Starfish Speed | MAC vector | -18.1% | -14.6% | 0.0% | +0.3% | +1.9% | -| Starfish Speed | ML-DSA-44 | -17.4% | -14.2% | +0.4% | -3.0% | +1.3% | -| Sparse-Starfish-Speed | Ed25519 | -24.5% | -19.6% | +0.2% | -4.6% | -8.2% | -| Sparse-Starfish-Speed | MAC vector | -25.2% | -21.0% | +0.1% | -2.3% | -11.3% | -| Sparse-Starfish-Speed | ML-DSA-44 | -24.1% | -19.4% | +0.4% | -4.3% | -6.6% | - -## Interpretation - -- Authentication did not materially change throughput or latency under the - emulated wide-area delays. Within each protocol, TPS differed by at most - 0.4%, block latency by at most 1.1%, and end-to-end latency by at most 1.4%. - These small changes are below what should be interpreted without repeated - trials and variance estimates. -- MAC vectors increased outbound bandwidth by 4.4-8.2% relative to Ed25519, - while ML-DSA-44 increased it by 51.0-57.8%. At the geo-limited block rate, - payload and protocol traffic dominate more of the total than in the - zero-latency runs, so ML-DSA's relative bandwidth multiplier is smaller. -- Starfish Speed reduced block latency by 16.8-18.1% and end-to-end latency by - 13.9-14.6% against matching plain-Starfish authentication, with essentially - identical TPS. -- Sparse-Starfish-Speed reduced block latency by 24.1-25.2%, end-to-end - latency by 19.4-21.0%, and outbound bandwidth by 6.6-11.3% against matching - plain-Starfish authentication, again with essentially identical TPS. -- All variants committed roughly 815-818 TPS from the offered 1,000 tx/s. - This experiment measures a local machine under injected network delay; it - does not establish capacity on physically distributed hardware. - -## Caveats - -- There is one run per configuration and no warm-up exclusion, so these are - directional comparisons rather than confidence intervals. -- The reported metrics were emitted before shutdown. Validator task abortion - logs expected `JoinError::Cancelled` messages afterward. Two runs also - printed a macOS `pthread lock` teardown error after their metrics; process - and benchmark-directory checks showed no active or overlapping benchmark. - The shutdown path should still be hardened before unattended batch runs. diff --git a/benchmark-results/2026-07-13-starfish-authentication-local.md b/benchmark-results/2026-07-13-starfish-authentication-local.md deleted file mode 100644 index b4d05f11..00000000 --- a/benchmark-results/2026-07-13-starfish-authentication-local.md +++ /dev/null @@ -1,104 +0,0 @@ -# Starfish authentication comparison — local Apple Silicon - -Date: 2026-07-13
-Source revision: `8a3bded` plus the Sparse authentication changes committed with this report
-Host: Apple Silicon (`arm64`), macOS 15.7.4
-Build: Rust 1.86.0, release profile - -## Configuration - -- 10 honest validators in one local process -- 1,000 tx/s offered load (100 tx/s per validator) -- 20-second measurement window -- Uniform 0 ms added network latency -- Default protocol dissemination mode (`push-useful` for all three protocols) -- One run per configuration - -Command template: - -```text -target/release/starfish local-benchmark \ - --committee-size 10 \ - --load 1000 \ - --consensus \ - --duration-secs 20 \ - --uniform-latency-ms 0 -``` - -## Results - -| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | -|---|---|---:|---:|---:|---:|---:|---:|---:| -| Starfish | Ed25519 | 12.1 | 13.9 | 446.50 | 4,573.30 | 14.33 | 14.30 | 65.74 | -| Starfish | MAC vector | 12.8 | 14.8 | 433.60 | 4,494.05 | 14.04 | 14.01 | 66.34 | -| Starfish | ML-DSA-44 | 15.7 | 18.0 | 432.50 | 4,125.75 | 57.11 | 57.08 | 270.44 | -| Starfish Speed | Ed25519 | 10.2 | 12.3 | 442.75 | 4,201.30 | 14.61 | 14.58 | 67.60 | -| Starfish Speed | MAC vector | 9.4 | 11.4 | 439.15 | 4,114.10 | 14.12 | 14.09 | 65.85 | -| Starfish Speed | ML-DSA-44 | 13.4 | 17.4 | 453.80 | 2,868.60 | 39.84 | 39.82 | 179.82 | -| Sparse-Starfish-Speed | Ed25519 | 5.5 | 10.8 | 430.00 | 4,867.50 | 8.99 | 8.96 | 42.84 | -| Sparse-Starfish-Speed | MAC vector | 5.2 | 10.5 | 430.75 | 5,211.95 | 9.73 | 9.69 | 46.25 | -| Sparse-Starfish-Speed | ML-DSA-44 | 5.5 | 10.1 | 440.05 | 4,335.90 | 37.85 | 37.82 | 176.16 | - -## Relative to Ed25519 within each protocol - -| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---|---:|---:|---:|---:|---:| -| Starfish | MAC vector | +5.8% | +6.5% | -2.9% | -1.7% | -2.0% | -| Starfish | ML-DSA-44 | +29.8% | +29.5% | -3.1% | -9.8% | +298.5% | -| Starfish Speed | MAC vector | -7.8% | -7.3% | -0.8% | -2.1% | -3.4% | -| Starfish Speed | ML-DSA-44 | +31.4% | +41.5% | +2.5% | -31.7% | +172.7% | -| Sparse-Starfish-Speed | MAC vector | -5.5% | -2.8% | +0.2% | +7.1% | +8.2% | -| Sparse-Starfish-Speed | ML-DSA-44 | 0.0% | -6.5% | +2.3% | -10.9% | +321.0% | - -## Starfish Speed relative to Starfish - -| Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---:|---:|---:|---:|---:| -| Ed25519 | -15.7% | -11.5% | -0.8% | -8.1% | +2.0% | -| MAC vector | -26.6% | -23.0% | +1.3% | -8.5% | +0.6% | -| ML-DSA-44 | -14.6% | -3.3% | +4.9% | -30.5% | -30.2% | - -## Sparse-Starfish-Speed relative to Starfish - -| Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---:|---:|---:|---:|---:| -| Ed25519 | -54.5% | -22.3% | -3.7% | +6.4% | -37.3% | -| MAC vector | -59.4% | -29.1% | -0.7% | +16.0% | -30.7% | -| ML-DSA-44 | -65.0% | -43.9% | +1.7% | +5.1% | -33.7% | - -## Interpretation - -- The Starfish and Starfish Speed MAC variants remained close to Ed25519: TPS - was within 3%, BPS within 2.1%, and bandwidth was slightly lower in this - sample. The latency changes are small enough that repeated runs are needed - before treating their sign as meaningful. -- ML-DSA-44 materially increased latency and bandwidth. Its signature is 2,420 - bytes, versus 64 bytes for Ed25519. Outbound bandwidth increased about 4.0x - for Starfish, 2.7x for Starfish Speed, and 4.2x for Sparse-Starfish-Speed. -- Starfish Speed with ML-DSA-44 produced 31.7% fewer blocks than its Ed25519 - variant while committing 2.5% more transactions. This indicates more - transactions per block in this run; it should not be read as evidence that - ML-DSA improves throughput without repeated trials. -- Against matching Starfish authentication variants, Starfish Speed had lower - latency in all three samples and essentially equal TPS for Ed25519 and MAC. - Its ML-DSA-44 run used about 30% less bandwidth, alongside about 30% fewer - blocks, than Starfish ML-DSA-44. -- Sparse-Starfish-Speed preserved roughly the same TPS as plain Starfish while - reducing outbound bandwidth by 31-37% and block latency by 55-65% across the - three authentication schemes. Its lean headers therefore remain beneficial - with either signatures or MACs in this local sample. -- Sparse MAC remained close to Sparse Ed25519: TPS differed by 0.2%, while - outbound bandwidth was 8.2% higher. Sparse ML-DSA-44 used 4.2x the outbound - bandwidth of Sparse Ed25519 despite Sparse's lower protocol overhead. -- All variants achieved roughly 430–454 TPS from the 1,000 tx/s offered load. - Ten validators share one laptop and therefore contend for the same CPU, - storage, and network stack. These results are useful for directional local - comparison, not distributed capacity claims. - -## Benchmark validation note - -During the first Sparse MAC run, the receiver-side transport guard exposed a -round-gap response that still carried full MAC vectors. The sender now routes -round-gap blocks through the same recipient-tag preparation used by relay and -missing-parent paths. The rejected run was discarded; the Sparse MAC row above -is the clean rerun, which emitted no transport rejections. diff --git a/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md b/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md deleted file mode 100644 index bdc746ae..00000000 --- a/benchmark-results/2026-07-14-authentication-geo-60-validators-60s.md +++ /dev/null @@ -1,165 +0,0 @@ -# Authentication comparison — 60-validator geographic emulation - -Date: 2026-07-14
-Source revisions: `4553e5e` (original matrix), `3a82c04` (ML-DSA-65 -extension)
-Host: Apple Silicon (`arm64`), macOS 15.7.4
-Build: Rust 1.86.0, release profile - -## Configuration - -- 60 honest validators in one local process -- 600 tx/s aggregate offered load (exactly 10 tx/s per validator) -- 60-second measurement window -- Protocol-default dissemination: `push-useful` for the Starfish families and - `pull` for Bluestreak -- One run per configuration -- Geographic latency emulation enabled; no uniform-latency override - -The harness has ten AWS region profiles. Validator indices map to profiles -modulo ten, so this experiment models six validators per region. RTT values -are divided by two to obtain one-way delays and independent ±3% per-message -jitter is applied. Base one-way delays range from 0.5 ms to 154.5 ms. - -This is a single-machine latency emulation, not a 60-host deployment. The -validators share CPU, memory, storage, loopback networking, and kernel socket -resources. They form 3,540 directed peer relationships and use one RocksDB -instance each. - -Command template: - -```text -target/release/starfish local-benchmark \ - --committee-size 60 \ - --load 600 \ - --consensus \ - --duration-secs 60 -``` - -## Results - -| Protocol | Authentication | Block latency (ms) | E2E latency (ms) | TPS | BPS | Bandwidth out (MB/s) | Bandwidth in (MB/s) | Bandwidth efficiency | -|---|---|---:|---:|---:|---:|---:|---:|---:| -| Starfish | Ed25519 | 664.45 | 728.77 | 444.30 | 515.02 | 8.59 | 8.58 | 39.59 | -| Starfish | MAC vector | 680.68 | 753.42 | 414.87 | 449.23 | 8.35 | 8.34 | 41.21 | -| Starfish | ML-DSA-44 | 849.38 | 937.02 | 427.77 | 417.95 | 9.72 | 9.71 | 46.53 | -| Starfish | ML-DSA-65† | 749.38 | 837.48 | 413.53 | 443.48 | 13.98 | 13.98 | 69.25 | -| Starfish Speed | Ed25519 | 592.13 | 673.87 | 460.37 | 467.12 | 6.63 | 6.62 | 29.49 | -| Starfish Speed | MAC vector | 670.80 | 783.93 | 430.95 | 439.30 | 7.30 | 7.30 | 34.70 | -| Starfish Speed | ML-DSA-44 | 641.70 | 726.03 | 433.63 | 461.57 | 11.06 | 11.05 | 52.22 | -| Starfish Speed | ML-DSA-65 | 641.40 | 727.28 | 421.98 | 415.88 | 12.10 | 12.09 | 58.71 | -| Sparse-Starfish-Speed | Ed25519 | 467.70 | 525.98 | 466.22 | 535.57 | 0.88 | 0.88 | 3.87 | -| Sparse-Starfish-Speed | MAC vector | 467.67 | 528.23 | 466.62 | 551.88 | 1.84 | 1.84 | 8.08 | -| Sparse-Starfish-Speed | ML-DSA-44 | 464.87 | 526.97 | 467.07 | 528.88 | 2.98 | 2.98 | 13.08 | -| Sparse-Starfish-Speed | ML-DSA-65 | 461.05 | 516.02 | 469.62 | 560.13 | 3.95 | 3.95 | 17.24 | -| Bluestreak | Ed25519 | 422.53 | 480.58 | 468.22 | 551.08 | 0.52 | 0.52 | 2.28 | -| Bluestreak | MAC vector | 421.62 | 480.37 | 467.63 | 545.72 | 1.47 | 1.46 | 6.42 | -| Bluestreak | ML-DSA-44 | 422.72 | 481.70 | 467.88 | 547.13 | 1.75 | 1.75 | 7.67 | -| Bluestreak | ML-DSA-65 | 423.98 | 482.82 | 470.25 | 548.02 | 2.22 | 2.22 | 9.66 | - -All sixteen commands exited successfully after emitting their metrics. The -Starfish ML-DSA-65 run marked † crossed this host's socket-buffer ceiling at -second 20: three writes reported `No buffer space available`, followed by -three decompress/deserialize warnings. Its metrics are retained as a -host-contended stress result, not a clean protocol comparison. The other -three ML-DSA-65 runs had no authentication, deserialize, socket-buffer, or -transport errors. - -## Warm-up-adjusted offered-rate utilization - -The transaction generators wait 13 seconds before submitting: the default -10-second delay plus 3 seconds for a 60-validator committee. The displayed TPS -uses the complete 60-second window, leaving approximately 47 active submission -seconds. The following values estimate the active rate as `TPS × 60 / 47`. - -| Protocol | Authentication | Estimated active TPS | Offered rate sustained | -|---|---|---:|---:| -| Starfish | Ed25519 | 567.19 | 94.5% | -| Starfish | MAC vector | 529.62 | 88.3% | -| Starfish | ML-DSA-44 | 546.09 | 91.0% | -| Starfish | ML-DSA-65† | 527.91 | 88.0% | -| Starfish Speed | Ed25519 | 587.71 | 98.0% | -| Starfish Speed | MAC vector | 550.15 | 91.7% | -| Starfish Speed | ML-DSA-44 | 553.57 | 92.3% | -| Starfish Speed | ML-DSA-65 | 538.70 | 89.8% | -| Sparse-Starfish-Speed | Ed25519 | 595.17 | 99.2% | -| Sparse-Starfish-Speed | MAC vector | 595.69 | 99.3% | -| Sparse-Starfish-Speed | ML-DSA-44 | 596.26 | 99.4% | -| Sparse-Starfish-Speed | ML-DSA-65 | 599.51 | 99.9% | -| Bluestreak | Ed25519 | 597.73 | 99.6% | -| Bluestreak | MAC vector | 596.97 | 99.5% | -| Bluestreak | ML-DSA-44 | 597.29 | 99.5% | -| Bluestreak | ML-DSA-65 | 600.32 | 100.1% | - -## Relative to Ed25519 within each protocol - -| Protocol | Authentication | Block latency | E2E latency | TPS | BPS | Bandwidth out | -|---|---|---:|---:|---:|---:|---:| -| Starfish | MAC vector | +2.4% | +3.4% | -6.6% | -12.8% | -2.8% | -| Starfish | ML-DSA-44 | +27.8% | +28.6% | -3.7% | -18.8% | +13.2% | -| Starfish | ML-DSA-65† | +12.8% | +14.9% | -6.9% | -13.9% | +62.7% | -| Starfish Speed | MAC vector | +13.3% | +16.3% | -6.4% | -6.0% | +10.1% | -| Starfish Speed | ML-DSA-44 | +8.4% | +7.7% | -5.8% | -1.2% | +66.8% | -| Starfish Speed | ML-DSA-65 | +8.3% | +7.9% | -8.3% | -11.0% | +82.5% | -| Sparse-Starfish-Speed | MAC vector | 0.0% | +0.4% | +0.1% | +3.0% | +109.1% | -| Sparse-Starfish-Speed | ML-DSA-44 | -0.6% | +0.2% | +0.2% | -1.2% | +238.6% | -| Sparse-Starfish-Speed | ML-DSA-65 | -1.4% | -1.9% | +0.7% | +4.6% | +348.9% | -| Bluestreak | MAC vector | -0.2% | 0.0% | -0.1% | -1.0% | +182.7% | -| Bluestreak | ML-DSA-44 | 0.0% | +0.2% | -0.1% | -0.7% | +236.5% | -| Bluestreak | ML-DSA-65 | +0.3% | +0.5% | +0.4% | -0.6% | +326.9% | - -Raw bandwidth can fall despite a larger authentication proof when a run -produces fewer blocks, as in plain Starfish MAC. The bandwidth-efficiency -metric—the ratio of bytes sent to committed transaction-payload bytes—rises -from 39.59 to 41.21 for MAC and 69.25 for ML-DSA-65, capturing the normalized -increase even when raw block production varies. - -## Interpretation - -- Sixty validators are feasible on this machine at a 600 tx/s aggregate - offered load, but headroom depends strongly on the protocol family. -- Sparse-Starfish-Speed and Bluestreak sustain 99.2-100.1% of the active - offered rate across all four authentication schemes. Within each family, - block latency varies by at most 1.4% and throughput by at most 0.7%. -- Plain Starfish sustains 88.0-94.5% of the active offered rate, while - Starfish Speed sustains 89.8-98.0%. Their authentication comparisons are - therefore partly measurements of shared-host contention. Plain Starfish - ML-DSA-65 is the clearest limit: its 13.98 MB/s full-mesh traffic triggered - the host's socket-buffer errors, so that row is not a clean protocol result. -- Bluestreak has the lowest latency and absolute bandwidth across every - authentication scheme: 422-424 ms block latency, 480-483 ms end-to-end - latency, and 0.52-2.22 MB/s outbound. -- Relative to matching Sparse-Starfish-Speed authentication, Bluestreak lowers - block latency by 8.0-9.8%, end-to-end latency by 6.4-9.1%, and outbound - bandwidth by 20.1-43.8%, with nearly identical throughput. -- At 60 validators, a full author MAC vector contains 60 × 32 = 1,920 bytes, - compared with a 64-byte Ed25519 signature and a 2,420-byte ML-DSA-44 - signature; an ML-DSA-65 signature is 3,309 bytes. ML-DSA-65 public keys are - 1,952 bytes and are provisioned in committee configuration rather than sent - in each block. Relays and synchronization responses for MAC blocks still - carry only one 32-byte recipient tag. Because Bluestreak and - Sparse-Starfish-Speed remove most other traffic, authentication bytes - produce large percentages while absolute bandwidth remains well below the - denser protocols. -- For repeatable all-protocol authentication comparisons on this single host, - 40 validators remains the safer configuration. Sixty validators is a useful - stress configuration and a clean operating point for the sparse and - Bluestreak families; denser families should move to multiple machines for - stronger conclusions. - -## Caveats - -- There is one sequential run per configuration, with no randomized order or - confidence interval. Host scheduling and thermal state can affect results. -- The ML-DSA-65 rows were appended after the original matrix on source - revision `3a82c04`. The marked plain-Starfish row encountered host transport - errors and should be treated only as evidence that this configuration - exceeds the local machine's clean operating point. -- The 600 tx/s load differs from the earlier 40-validator 1,000 tx/s matrix; - the two experiments should not be treated as a pure committee-size scaling - comparison. -- Six validators share each synthetic AWS region profile. The harness injects - the delay distribution but does not model independent machines or WAN - bandwidth constraints. -- Displayed TPS includes the 13-second startup delay. Warm-up-adjusted TPS is - derived rather than measured in a separately gated metrics window. diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index 2dbf4077..39afa7b3 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -129,7 +129,13 @@ each load generator submits a fixed load of 100 tx/s or more precisely 10 tx every 100ms. Performance measurements are collected by regularly scraping the Prometheus metrics exposed by the load generators. -Available consensus protocols: `starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-ml-dsa-65`, `starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, `starfish-speed-ml-dsa-65`, `sparse-starfish-speed`, `sparse-starfish-speed-mac`, `sparse-starfish-speed-ml-dsa-44`, `sparse-starfish-speed-ml-dsa-65`, `bluestreak`, `bluestreak-mac`, `bluestreak-ml-dsa-44`, `bluestreak-ml-dsa-65`, `starfish-bls`, `mysticeti`, `mysticeti-bls`, `cordial-miners`, `sailfish-pp`. +Available consensus protocols: `starfish`, `starfish-speed`, +`sparse-starfish-speed`, `bluestreak`, `starfish-bls`, `mysticeti`, +`mysticeti-bls`, `cordial-miners`, and `sailfish-pp`. Select the block signature +for any protocol with `--block-authentication ed25519|ml-dsa-44|ml-dsa-65`; +Ed25519 is the default. The `starfish-mac`, `starfish-speed-mac`, +`sparse-starfish-speed-mac`, and `bluestreak-mac` names are separate +experimental protocols and cannot be combined with that option. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index 68487b56..cec244f1 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -54,15 +54,8 @@ pub struct BenchmarkParametersGeneric { /// single VPC, they should use their internal IPs to avoid /// paying for data sent between the nodes. pub use_internal_ip_address: bool, - // Consensus protocol to deploy - // (starfish | starfish-mac | starfish-ml-dsa-44 | starfish-ml-dsa-65 | - // starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | - // starfish-speed-ml-dsa-65 | - // sparse-starfish-speed | sparse-starfish-speed-mac | - // sparse-starfish-speed-ml-dsa-44 | sparse-starfish-speed-ml-dsa-65 | - // bluestreak | bluestreak-mac | bluestreak-ml-dsa-44 | - // bluestreak-ml-dsa-65 | starfish-bls | mysticeti | mysticeti-bls | - // cordial-miners | sailfish-pp) + /// Consensus protocol to deploy. The block signature is configured in + /// `node_parameters`; the `*-mac` names denote experimental protocols. pub consensus_protocol: String, /// number Byzantine nodes pub byzantine_nodes: usize, diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index fc2cf3f6..ebd04de2 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -58,6 +58,11 @@ pub struct Opts { )] settings_path: String, + /// Block signature scheme used by every selected consensus protocol. + /// Defaults to Ed25519. Not applicable to experimental `*-mac` protocols. + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65", global = true)] + block_authentication: Option, + /// The type of operation to run. #[clap(subcommand)] operation: Operation, @@ -134,14 +139,9 @@ pub enum Operation { #[clap(long, action, default_value_t = false, global = true)] skip_testbed_configuration: bool, - /// Protocols to benchmark in order. Available options: - /// starfish | starfish-mac | starfish-ml-dsa-44 | starfish-ml-dsa-65 | - /// starfish-speed | starfish-speed-mac | starfish-speed-ml-dsa-44 | starfish-speed-ml-dsa-65 | - /// sparse-starfish-speed | sparse-starfish-speed-mac | - /// sparse-starfish-speed-ml-dsa-44 | sparse-starfish-speed-ml-dsa-65 | - /// bluestreak | bluestreak-mac | bluestreak-ml-dsa-44 | bluestreak-ml-dsa-65 | - /// starfish-bls | mysticeti | mysticeti-bls | - /// cordial-miners | sailfish-pp + /// Consensus protocols to benchmark in order. The `*-mac` names are + /// separate experimental protocols; signature schemes are selected + /// with `--block-authentication`. #[clap( long, value_name = "STRING", @@ -851,6 +851,7 @@ fn load_benchmark_configs( dissemination_mode: &Option, compress_network: Option, bls_workers: Option, + block_authentication: &Option, ) -> eyre::Result<(NodeParameters, ClientParameters)> { let mut node_parameters = match &settings.node_parameters_path { Some(path) => NodeParameters::load(path).wrap_err("Failed to load node's parameters")?, @@ -858,6 +859,9 @@ fn load_benchmark_configs( }; node_parameters.adversarial_latency = adversarial_latency; node_parameters.adversarial_latency_percent = adversarial_latency_percent; + if block_authentication.is_some() { + node_parameters.block_authentication = block_authentication.clone(); + } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1037,6 +1041,7 @@ async fn run( .await .wrap_err("Failed to crate testbed")?; + let block_authentication = opts.block_authentication.clone(); match opts.operation { Operation::Testbed { action } => match action { // Display the current status of the testbed. @@ -1233,6 +1238,7 @@ async fn run( &dissemination_mode, compress_network, resolved_bls_workers.override_workers, + &block_authentication, )?; display::newline(); @@ -1399,6 +1405,7 @@ async fn run( &dissemination_mode, compress_network, resolved_bls_workers.override_workers, + &block_authentication, )?; display::newline(); @@ -1605,6 +1612,7 @@ async fn run( &dissemination_mode, compress_network, resolved_bls_workers.override_workers, + &block_authentication, )?; display::newline(); @@ -1770,6 +1778,7 @@ async fn run( &dissemination_mode, compress_network, resolved_bls_workers.override_workers, + &block_authentication, )?; display::newline(); @@ -1975,6 +1984,7 @@ async fn run( &dissemination_mode, compress_network, resolved_bls_workers.override_workers, + &block_authentication, )?; display::newline(); @@ -2283,6 +2293,8 @@ mod tests { let opts = Opts::try_parse_from([ "orchestrator", "benchmark", + "--block-authentication", + "ml-dsa-65", "--protocols", "starfish", "mysticeti", @@ -2294,6 +2306,8 @@ mod tests { ]) .unwrap(); + assert_eq!(opts.block_authentication.as_deref(), Some("ml-dsa-65")); + match opts.operation { Operation::Benchmark { protocols, diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 5ac24437..e453d5ea 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -57,6 +57,10 @@ pub struct NodeParameters { pub bls_verification_workers: usize, #[serde(default)] pub dissemination_mode: DisseminationMode, + /// Block signature scheme. `None` selects Ed25519. Experimental MAC + /// protocols select their authentication through the consensus name. + #[serde(default)] + pub block_authentication: Option, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -130,6 +134,7 @@ impl Default for NodeParameters { compress_network: node_defaults::default_compress_network(), bls_verification_workers: node_defaults::default_bls_verification_workers(), dissemination_mode: DisseminationMode::default(), + block_authentication: None, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 2c282194..602ecafb 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -303,30 +303,53 @@ pub struct ProtocolConfig { impl ProtocolConfig { pub fn from_str(value: &str) -> Result { - let (protocol_name, block_authentication_scheme) = [ - ("-ml-dsa-65", BlockAuthenticationScheme::MlDsa65), - ("-ml-dsa-44", BlockAuthenticationScheme::MlDsa44), - ("-mac", BlockAuthenticationScheme::MacVector), - ] - .into_iter() - .find_map(|(suffix, scheme)| value.strip_suffix(suffix).map(|base| (base, scheme))) - .unwrap_or((value, BlockAuthenticationScheme::Ed25519)); + Self::from_selection(value, None) + } + pub fn from_selection( + consensus: &str, + block_authentication: Option<&str>, + ) -> Result { + let (protocol_name, is_mac_experiment) = consensus + .strip_suffix("-mac") + .map(|base| (base, true)) + .unwrap_or((consensus, false)); let consensus_protocol = ConsensusProtocol::from_known_str(protocol_name) - .ok_or_else(|| format!("Unknown consensus protocol '{value}'"))?; - if block_authentication_scheme != BlockAuthenticationScheme::Ed25519 - && !matches!( + .ok_or_else(|| format!("Unknown consensus protocol '{consensus}'"))?; + + let block_authentication_scheme = if is_mac_experiment { + if block_authentication.is_some() { + return Err(format!( + "'{consensus}' is an experimental MAC protocol and cannot be combined with \ + --block-authentication" + )); + } + if !matches!( consensus_protocol, ConsensusProtocol::Starfish | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed | ConsensusProtocol::Bluestreak - ) - { - return Err(format!( - "Block authentication variants are not supported for '{protocol_name}'" - )); - } + ) { + return Err(format!( + "The experimental MAC protocol is not available for '{protocol_name}'" + )); + } + BlockAuthenticationScheme::MacVector + } else { + match block_authentication.unwrap_or("ed25519") { + "ed25519" => BlockAuthenticationScheme::Ed25519, + "ml-dsa-44" => BlockAuthenticationScheme::MlDsa44, + "ml-dsa-65" => BlockAuthenticationScheme::MlDsa65, + value => { + return Err(format!( + "Unknown block authentication scheme '{value}'. Use 'ed25519', \ + 'ml-dsa-44', or 'ml-dsa-65'." + )); + } + } + }; + Ok(Self { consensus_protocol, block_authentication_scheme, @@ -565,6 +588,32 @@ impl DagState { storage_backend: &StorageBackend, strong_vote_adaptive_acknowledgments: bool, dissemination_mode: DisseminationMode, + ) -> RecoveredState { + let protocol_config = ProtocolConfig::from_str(&consensus).expect("validated protocol"); + Self::open_with_protocol_config( + authority, + path, + metrics, + committee, + byzantine_strategy, + protocol_config, + storage_backend, + strong_vote_adaptive_acknowledgments, + dissemination_mode, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn open_with_protocol_config( + authority: AuthorityIndex, + path: impl AsRef, + metrics: Arc, + committee: Arc, + byzantine_strategy: String, + protocol_config: ProtocolConfig, + storage_backend: &StorageBackend, + strong_vote_adaptive_acknowledgments: bool, + dissemination_mode: DisseminationMode, ) -> RecoveredState { assert!( committee.len() <= crate::types::MAX_COMMITTEE_SIZE as usize, @@ -588,7 +637,6 @@ impl DagState { Arc::new(RocksStore::open(&path).expect("Failed to open RocksDB")) } }; - let protocol_config = ProtocolConfig::from_str(&consensus).expect("validated protocol"); let consensus_protocol = protocol_config.consensus_protocol; let resolved_dissemination = consensus_protocol.resolve_dissemination_mode(dissemination_mode); @@ -4933,122 +4981,60 @@ mod tests { #[test] fn protocol_config_selects_block_authentication() { - assert_eq!( - ProtocolConfig::from_str("starfish").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Starfish, - block_authentication_scheme: BlockAuthenticationScheme::Ed25519, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-mac").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Starfish, - block_authentication_scheme: BlockAuthenticationScheme::MacVector, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-ml-dsa-44").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Starfish, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-ml-dsa-65").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Starfish, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-speed").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::StarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::Ed25519, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-speed-mac").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::StarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::MacVector, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-speed-ml-dsa-44").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::StarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, - } - ); - assert_eq!( - ProtocolConfig::from_str("starfish-speed-ml-dsa-65").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::StarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, - } - ); - assert_eq!( - ProtocolConfig::from_str("sparse-starfish-speed").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::Ed25519, - } - ); - assert_eq!( - ProtocolConfig::from_str("sparse-starfish-speed-mac").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::MacVector, - } - ); - assert_eq!( - ProtocolConfig::from_str("sparse-starfish-speed-ml-dsa-44").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, - } - ); - assert_eq!( - ProtocolConfig::from_str("sparse-starfish-speed-ml-dsa-65").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::SparseStarfishSpeed, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, - } - ); - assert_eq!( - ProtocolConfig::from_str("bluestreak").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Bluestreak, - block_authentication_scheme: BlockAuthenticationScheme::Ed25519, - } - ); - assert_eq!( - ProtocolConfig::from_str("bluestreak-mac").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Bluestreak, - block_authentication_scheme: BlockAuthenticationScheme::MacVector, - } - ); - assert_eq!( - ProtocolConfig::from_str("bluestreak-ml-dsa-44").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Bluestreak, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa44, - } - ); - assert_eq!( - ProtocolConfig::from_str("bluestreak-ml-dsa-65").unwrap(), - ProtocolConfig { - consensus_protocol: ConsensusProtocol::Bluestreak, - block_authentication_scheme: BlockAuthenticationScheme::MlDsa65, + let protocols = [ + ("mysticeti", ConsensusProtocol::Mysticeti), + ("cordial-miners", ConsensusProtocol::CordialMiners), + ("starfish", ConsensusProtocol::Starfish), + ("starfish-speed", ConsensusProtocol::StarfishSpeed), + ("starfish-bls", ConsensusProtocol::StarfishBls), + ("sailfish-pp", ConsensusProtocol::SailfishPlusPlus), + ("bluestreak", ConsensusProtocol::Bluestreak), + ("mysticeti-bls", ConsensusProtocol::MysticetiBls), + ( + "sparse-starfish-speed", + ConsensusProtocol::SparseStarfishSpeed, + ), + ]; + let signature_schemes = [ + (None, BlockAuthenticationScheme::Ed25519), + (Some("ed25519"), BlockAuthenticationScheme::Ed25519), + (Some("ml-dsa-44"), BlockAuthenticationScheme::MlDsa44), + (Some("ml-dsa-65"), BlockAuthenticationScheme::MlDsa65), + ]; + + for (name, consensus_protocol) in protocols { + for (selection, block_authentication_scheme) in signature_schemes { + assert_eq!( + ProtocolConfig::from_selection(name, selection).unwrap(), + ProtocolConfig { + consensus_protocol, + block_authentication_scheme, + } + ); } - ); - assert!(ProtocolConfig::from_str("mysticeti-ml-dsa-65").is_err()); + } + + for (name, consensus_protocol) in [ + ("starfish-mac", ConsensusProtocol::Starfish), + ("starfish-speed-mac", ConsensusProtocol::StarfishSpeed), + ( + "sparse-starfish-speed-mac", + ConsensusProtocol::SparseStarfishSpeed, + ), + ("bluestreak-mac", ConsensusProtocol::Bluestreak), + ] { + assert_eq!( + ProtocolConfig::from_str(name).unwrap(), + ProtocolConfig { + consensus_protocol, + block_authentication_scheme: BlockAuthenticationScheme::MacVector, + } + ); + assert!(ProtocolConfig::from_selection(name, Some("ed25519")).is_err()); + } + + assert!(ProtocolConfig::from_str("mysticeti-mac").is_err()); + assert!(ProtocolConfig::from_selection("starfish", Some("unknown")).is_err()); assert!(ProtocolConfig::from_str("starfish-unknown").is_err()); - assert!(ProtocolConfig::from_str("starfish-speed-unknown").is_err()); - assert!(ProtocolConfig::from_str("sparse-starfish-speed-unknown").is_err()); - assert!(ProtocolConfig::from_str("bluestreak-unknown").is_err()); } } diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 0b234c4c..ec9942eb 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -59,8 +59,8 @@ const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; /// Enforce the MAC experiment's transport contract before cryptographic /// verification: /// -/// - a full vector is accepted only on proactive block streaming directly -/// from the block's claimed author; +/// - a full vector is accepted only on proactive block streaming directly from +/// the block's claimed author; /// - every relay and synchronization path must carry one recipient tag; /// - a direct author stream must carry the full vector, so recipients retain /// the material needed for one-hop relay. diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index f5b06f4b..123c872d 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -45,7 +45,11 @@ impl Validator { byzantine_strategy: String, consensus: String, ) -> Result { - let protocol_config = ProtocolConfig::from_str(&consensus).map_err(|error| eyre!(error))?; + let protocol_config = ProtocolConfig::from_selection( + &consensus, + public_config.parameters.block_authentication.as_deref(), + ) + .map_err(|error| eyre!(error))?; match protocol_config.block_authentication_scheme { BlockAuthenticationScheme::Ed25519 => { if committee.get_public_key(authority) != Some(&private_config.keypair.public_key()) @@ -139,13 +143,13 @@ impl Validator { // Open the DAG state. let rocks_path = private_config.rocksdb(); - let recovered = DagState::open( + let recovered = DagState::open_with_protocol_config( authority, rocks_path, metrics.clone(), committee.clone(), byzantine_strategy, - consensus, + protocol_config, ¶meters.storage_backend, public_config .parameters @@ -299,11 +303,16 @@ mod smoke_tests { } } - async fn run_commit_test(consensus: &str, port_offset: u16) { + async fn run_commit_test( + consensus: &str, + block_authentication: Option<&str>, + port_offset: u16, + ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); - let public_config = + let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); + public_config.parameters.block_authentication = block_authentication.map(str::to_string); let parameters = Parameters::default(); let dir = TempDir::new().unwrap(); @@ -347,42 +356,52 @@ mod smoke_tests { } } - #[test_case("mysticeti", 0)] - #[test_case("cordial-miners", 40)] - #[test_case("starfish", 60)] - #[test_case("starfish-mac", 700)] - #[test_case("starfish-ml-dsa-44", 720)] - #[test_case("starfish-ml-dsa-65", 1000)] - #[test_case("starfish-speed", 80)] - #[test_case("starfish-speed-mac", 760)] - #[test_case("starfish-speed-ml-dsa-44", 780)] - #[test_case("starfish-speed-ml-dsa-65", 1040)] - #[test_case("starfish-bls", 100)] - #[test_case("sailfish++", 120)] - #[test_case("bluestreak", 140)] - #[test_case("mysticeti-bls", 160)] - #[test_case("sparse-starfish-speed", 180)] - #[test_case("sparse-starfish-speed-mac", 840)] - #[test_case("sparse-starfish-speed-ml-dsa-44", 860)] - #[test_case("sparse-starfish-speed-ml-dsa-65", 1080)] - #[test_case("bluestreak-mac", 920)] - #[test_case("bluestreak-ml-dsa-44", 940)] - #[test_case("bluestreak-ml-dsa-65", 1120)] + #[test_case("mysticeti", None, 0)] + #[test_case("mysticeti", Some("ml-dsa-65"), 1280)] + #[test_case("cordial-miners", None, 40)] + #[test_case("cordial-miners", Some("ml-dsa-65"), 1300)] + #[test_case("starfish", None, 60)] + #[test_case("starfish-mac", None, 700)] + #[test_case("starfish", Some("ml-dsa-44"), 720)] + #[test_case("starfish", Some("ml-dsa-65"), 1000)] + #[test_case("starfish-speed", None, 80)] + #[test_case("starfish-speed-mac", None, 760)] + #[test_case("starfish-speed", Some("ml-dsa-44"), 780)] + #[test_case("starfish-speed", Some("ml-dsa-65"), 1040)] + #[test_case("starfish-bls", None, 100)] + #[test_case("starfish-bls", Some("ml-dsa-65"), 1320)] + #[test_case("sailfish++", None, 120)] + #[test_case("sailfish++", Some("ml-dsa-65"), 1340)] + #[test_case("bluestreak", None, 140)] + #[test_case("mysticeti-bls", None, 160)] + #[test_case("mysticeti-bls", Some("ml-dsa-65"), 1360)] + #[test_case("sparse-starfish-speed", None, 180)] + #[test_case("sparse-starfish-speed-mac", None, 840)] + #[test_case("sparse-starfish-speed", Some("ml-dsa-44"), 860)] + #[test_case("sparse-starfish-speed", Some("ml-dsa-65"), 1080)] + #[test_case("bluestreak-mac", None, 920)] + #[test_case("bluestreak", Some("ml-dsa-44"), 940)] + #[test_case("bluestreak", Some("ml-dsa-65"), 1120)] #[tokio::test] - async fn validator_commit(consensus: &str, port_offset: u16) { - run_commit_test(consensus, port_offset).await; + async fn validator_commit( + consensus: &str, + block_authentication: Option<&str>, + port_offset: u16, + ) { + run_commit_test(consensus, block_authentication, port_offset).await; } #[tokio::test] async fn validator_commit_bluestreak_basic() { - run_commit_test("bluestreak", 150).await; + run_commit_test("bluestreak", None, 150).await; } - async fn run_sync_test(consensus: &str, port_offset: u16) { + async fn run_sync_test(consensus: &str, block_authentication: Option<&str>, port_offset: u16) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); - let public_config = + let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); + public_config.parameters.block_authentication = block_authentication.map(str::to_string); let parameters = Parameters::default(); let dir = TempDir::new().unwrap(); @@ -459,30 +478,30 @@ mod smoke_tests { } } - #[test_case("mysticeti", 100)] - #[test_case("cordial-miners", 140)] - #[test_case("starfish", 160)] - #[test_case("starfish-mac", 740)] - #[test_case("starfish-ml-dsa-44", 1020)] - #[test_case("starfish-ml-dsa-65", 1200)] - #[test_case("starfish-speed", 180)] - #[test_case("starfish-speed-mac", 800)] - #[test_case("starfish-speed-ml-dsa-44", 820)] - #[test_case("starfish-speed-ml-dsa-65", 1220)] - #[test_case("starfish-bls", 200)] - #[test_case("sailfish++", 220)] - #[test_case("bluestreak", 260)] - #[test_case("mysticeti-bls", 280)] - #[test_case("sparse-starfish-speed", 320)] - #[test_case("sparse-starfish-speed-mac", 880)] - #[test_case("sparse-starfish-speed-ml-dsa-44", 900)] - #[test_case("sparse-starfish-speed-ml-dsa-65", 1240)] - #[test_case("bluestreak-mac", 960)] - #[test_case("bluestreak-ml-dsa-44", 980)] - #[test_case("bluestreak-ml-dsa-65", 1260)] + #[test_case("mysticeti", None, 100)] + #[test_case("cordial-miners", None, 140)] + #[test_case("starfish", None, 160)] + #[test_case("starfish-mac", None, 740)] + #[test_case("starfish", Some("ml-dsa-44"), 1020)] + #[test_case("starfish", Some("ml-dsa-65"), 1200)] + #[test_case("starfish-speed", None, 180)] + #[test_case("starfish-speed-mac", None, 800)] + #[test_case("starfish-speed", Some("ml-dsa-44"), 820)] + #[test_case("starfish-speed", Some("ml-dsa-65"), 1220)] + #[test_case("starfish-bls", None, 200)] + #[test_case("sailfish++", None, 220)] + #[test_case("bluestreak", None, 260)] + #[test_case("mysticeti-bls", None, 280)] + #[test_case("sparse-starfish-speed", None, 320)] + #[test_case("sparse-starfish-speed-mac", None, 880)] + #[test_case("sparse-starfish-speed", Some("ml-dsa-44"), 900)] + #[test_case("sparse-starfish-speed", Some("ml-dsa-65"), 1240)] + #[test_case("bluestreak-mac", None, 960)] + #[test_case("bluestreak", Some("ml-dsa-44"), 980)] + #[test_case("bluestreak", Some("ml-dsa-65"), 1260)] #[tokio::test] - async fn validator_sync(consensus: &str, port_offset: u16) { - run_sync_test(consensus, port_offset).await; + async fn validator_sync(consensus: &str, block_authentication: Option<&str>, port_offset: u16) { + run_sync_test(consensus, block_authentication, port_offset).await; } async fn run_crash_faults_test(consensus: &str, port_offset: u16) { diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index ee02e1c5..73a6288c 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -65,10 +65,13 @@ enum Operation { parameters_path: String, #[clap(long, value_name = "STRING", default_value = "")] byzantine_strategy: String, - /// Consensus/authentication variant (for example `starfish-mac`, - /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-65`). + /// Consensus protocol. The `*-mac` names are experimental protocols. #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, + /// Block signature scheme. Defaults to Ed25519 and is not applicable + /// to the experimental `*-mac` protocols. + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65")] + block_authentication: Option, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -95,10 +98,13 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, - /// Consensus/authentication variant (for example `starfish-mac`, - /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-65`). + /// Consensus protocol. The `*-mac` names are experimental protocols. #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, + /// Block signature scheme. Defaults to Ed25519 and is not applicable + /// to the experimental `*-mac` protocols. + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65")] + block_authentication: Option, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -147,10 +153,13 @@ enum Operation { /// `--adversarial-latency` is enabled (0-100). #[clap(long, value_name = "INT", default_value_t = 34)] adversarial_latency_percent: u32, - /// Consensus/authentication variant (for example `starfish-mac`, - /// `bluestreak-mac`, or `sparse-starfish-speed-ml-dsa-65`). + /// Consensus protocol. The `*-mac` names are experimental protocols. #[clap(long, value_name = "STRING", default_value = "starfish")] consensus: String, + /// Block signature scheme. Defaults to Ed25519 and is not applicable + /// to the experimental `*-mac` protocols. + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65")] + block_authentication: Option, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, /// Dissemination mode override: @@ -184,6 +193,7 @@ async fn main() -> Result<()> { parameters_path, byzantine_strategy, consensus: consensus_protocol, + block_authentication, } => { run( authority, @@ -193,6 +203,7 @@ async fn main() -> Result<()> { parameters_path, byzantine_strategy, consensus_protocol, + block_authentication, ) .await? } @@ -206,6 +217,7 @@ async fn main() -> Result<()> { adversarial_latency, adversarial_latency_percent, consensus: consensus_protocol, + block_authentication, data_dir, base_ip, storage_backend, @@ -224,6 +236,7 @@ async fn main() -> Result<()> { adversarial_latency, adversarial_latency_percent, consensus_protocol, + block_authentication, data_dir, base_ip, storage_backend, @@ -244,6 +257,7 @@ async fn main() -> Result<()> { adversarial_latency, adversarial_latency_percent, consensus: consensus_protocol, + block_authentication, duration_secs, dissemination_mode, } => { @@ -253,6 +267,7 @@ async fn main() -> Result<()> { } node_parameters.adversarial_latency = adversarial_latency; node_parameters.adversarial_latency_percent = adversarial_latency_percent; + node_parameters.block_authentication = block_authentication; if let Some(ref mode) = dissemination_mode { node_parameters.dissemination_mode = parse_dissemination_mode(mode)?; } @@ -345,6 +360,17 @@ async fn local_benchmark( } println!("Transaction Load: {load} tx/s"); println!("Consensus Protocol: {consensus_protocol}"); + println!( + "Block Authentication: {}", + if consensus_protocol.ends_with("-mac") { + "mac-vector (experimental)" + } else { + node_parameters + .block_authentication + .as_deref() + .unwrap_or("ed25519") + } + ); if let Some(latency) = node_parameters.uniform_latency_ms { println!("Network Latency: {latency} ms (uniform)"); } else { @@ -517,14 +543,18 @@ async fn run( parameters_path: String, byzantine_strategy: String, consensus_protocol: String, + block_authentication: Option, ) -> Result<()> { tracing::info!("Starting node {authority}"); let committee = Committee::load(&committee_path) .wrap_err(format!("Failed to load committee file '{committee_path}'"))?; - let public_config = NodePublicConfig::load(&public_config_path).wrap_err(format!( + let mut public_config = NodePublicConfig::load(&public_config_path).wrap_err(format!( "Failed to load parameters file '{public_config_path}'" ))?; + if block_authentication.is_some() { + public_config.parameters.block_authentication = block_authentication; + } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" ))?; @@ -561,6 +591,7 @@ async fn dryrun( adversarial_latency: bool, adversarial_latency_percent: u32, consensus_protocol: String, + block_authentication: Option, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -602,6 +633,7 @@ async fn dryrun( node_parameters.adversarial_latency = adversarial_latency; node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.compress_network = compress_network; + node_parameters.block_authentication = block_authentication; if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -710,7 +742,9 @@ pub fn default_table_format() -> format::TableFormat { mod tests { use std::net::Ipv4Addr; - use super::ipv4_add_offset; + use clap::Parser; + + use super::{Args, Operation, ipv4_add_offset}; #[test] fn ipv4_add_offset_crosses_octet_boundary() { @@ -726,4 +760,32 @@ mod tests { let base = Ipv4Addr::new(255, 255, 255, 255); assert!(ipv4_add_offset(base, 1).is_err()); } + + #[test] + fn dry_run_parses_block_authentication_separately_from_consensus() { + let args = Args::try_parse_from([ + "starfish", + "dry-run", + "--authority", + "0", + "--committee-size", + "4", + "--consensus", + "mysticeti", + "--block-authentication", + "ml-dsa-65", + ]) + .unwrap(); + + let Operation::DryRun { + consensus, + block_authentication, + .. + } = args.operation + else { + panic!("expected dry-run operation"); + }; + assert_eq!(consensus, "mysticeti"); + assert_eq!(block_authentication.as_deref(), Some("ml-dsa-65")); + } } diff --git a/local-dryrun/README.md b/local-dryrun/README.md index 8b4e0b7b..5b568c2e 100644 --- a/local-dryrun/README.md +++ b/local-dryrun/README.md @@ -30,18 +30,16 @@ NUM_NODES=10 DESIRED_TPS=1000 CONSENSUS=starfish \ | `NUM_CRASHED_NODES` | `0` | Number of highest-numbered authorities to leave down from startup | | `DESIRED_TPS` | `1000` | Target transactions per second (split evenly across nodes) | | `CONSENSUS` | `bluestreak` | Consensus protocol (see below) | +| `BLOCK_AUTHENTICATION` | *(unset)* | `ed25519`, `ml-dsa-44`, or `ml-dsa-65`; unset defaults to Ed25519 | | `TEST_TIME` | `3000` | Experiment duration in seconds | -Supported `CONSENSUS` values include the Ed25519, MAC, ML-DSA-44, and ML-DSA-65 variants -of Starfish, Starfish Speed, Sparse-Starfish-Speed, and Bluestreak: -`starfish`, `starfish-mac`, `starfish-ml-dsa-44`, `starfish-ml-dsa-65`, -`starfish-speed`, `starfish-speed-mac`, `starfish-speed-ml-dsa-44`, -`starfish-speed-ml-dsa-65`, -`sparse-starfish-speed`, `sparse-starfish-speed-mac`, -`sparse-starfish-speed-ml-dsa-44`, `sparse-starfish-speed-ml-dsa-65`, -`bluestreak`, `bluestreak-mac`, `bluestreak-ml-dsa-44`, -`bluestreak-ml-dsa-65`, `starfish-bls`, `cordial-miners`, `mysticeti`, -`sailfish-pp`, and `mysticeti-bls`. +Supported `CONSENSUS` values are `starfish`, `starfish-speed`, +`sparse-starfish-speed`, `bluestreak`, `starfish-bls`, `cordial-miners`, +`mysticeti`, `sailfish-pp`, and `mysticeti-bls`. `BLOCK_AUTHENTICATION` +selects Ed25519, ML-DSA-44, or ML-DSA-65 for any of them. The +`starfish-mac`, `starfish-speed-mac`, `sparse-starfish-speed-mac`, and +`bluestreak-mac` names are separate experimental protocols; leave +`BLOCK_AUTHENTICATION` unset when using one. ### Protocol Tuning diff --git a/local-dryrun/dryrun.sh b/local-dryrun/dryrun.sh index 117404d1..3ca409f0 100755 --- a/local-dryrun/dryrun.sh +++ b/local-dryrun/dryrun.sh @@ -6,11 +6,10 @@ NUM_NODES=${NUM_NODES:-10} NUM_CRASHED_NODES=${NUM_CRASHED_NODES:-0} DESIRED_TPS=${DESIRED_TPS:-100} -# Authentication variants append -mac, -ml-dsa-44, or -ml-dsa-65 to starfish, -# starfish-speed, sparse-starfish-speed, or bluestreak. -# Other options: starfish-bls, cordial-miners, mysticeti, -# sailfish-pp, mysticeti-bls +# Signature schemes: ed25519 (default), ml-dsa-44, ml-dsa-65. +# The *-mac names remain separate experimental protocols. CONSENSUS=${CONSENSUS:- sparse-starfish-speed} +BLOCK_AUTHENTICATION=${BLOCK_AUTHENTICATION:-} NUM_BYZANTINE_NODES=${NUM_BYZANTINE_NODES:-0} # Options: timeout-leader, leader-withholding, # equivocating-chains, equivocating-two-chains, @@ -574,6 +573,10 @@ EOH if [ "${COMPRESS_NETWORK:-0}" = 1 ]; then PARAM_FLAGS+=" --compress-network" fi + if [ -n "$BLOCK_AUTHENTICATION" ]; then + PARAM_FLAGS+=" --block-authentication" + PARAM_FLAGS+=" $BLOCK_AUTHENTICATION" + fi cat < Date: Tue, 14 Jul 2026 19:07:21 +0200 Subject: [PATCH 19/62] Fix dependency policy checks for ML-DSA --- deny.toml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index 3f76a6c3..8ce45edd 100644 --- a/deny.toml +++ b/deny.toml @@ -28,7 +28,7 @@ allow = [ "BSD-2-Clause", "BSD-3-Clause", "CC0-1.0", - "GPL-2.0", + "GPL-2.0-or-later", "ISC", "LicenseRef-ring", "MIT", @@ -63,6 +63,14 @@ name = "minibytes" expression = "MIT" license-files = [{ path = "LICENSE", hash = 0x6b2d3210 }] +# bloom 0.3.2 uses the deprecated SPDX identifier `GPL-2.0`; its license +# grants redistribution under GPL version 2 or any later version. +[[licenses.clarify]] +name = "bloom" +version = "0.3.2" +expression = "GPL-2.0-or-later" +license-files = [{ path = "LICENSE", hash = 0xeaa66bfd }] + [licenses.private] # If true, ignores workspace crates that aren't published, or are only # published to private registries. @@ -90,6 +98,10 @@ skip = [ # tidehunter uses prometheus 0.14 (protobuf 3.x), starfish-core uses 0.13 (protobuf 2.x) { name = "prometheus", version = "0.13" }, { name = "protobuf", version = "2" }, + # ml-dsa 0.1 uses RustCrypto digest 0.11 and crypto-common 0.2 while + # existing workspace dependencies still require their previous releases. + { name = "crypto-common", version = "0.2" }, + { name = "digest", version = "0.11" }, ] skip-tree = [ # aws-smithy-http-client depends on both hyper 0.14 and 1.x, From 10287e686b1d1df302a8c934588d58cb14dd4823 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:42:24 +0200 Subject: [PATCH 20/62] Fix current stable Clippy lint --- crates/starfish-core/src/crypto.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 99735e8f..ea73416c 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -580,7 +580,7 @@ impl AsRef<[u8]> for MacTag { impl fmt::Debug for MacTag { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Mac({})", &hex::encode(&self.0[..4])) + write!(f, "Mac({})", hex::encode(&self.0[..4])) } } From fd4ee0f5a4c47ace6682983ed9cfb602f3d7b4c5 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:43:24 +0200 Subject: [PATCH 21/62] Document Starfish-RBC protocol design --- docs/starfish-rbc-protocol.md | 647 ++++++++++++++++++++++++++++++++++ 1 file changed, 647 insertions(+) create mode 100644 docs/starfish-rbc-protocol.md diff --git a/docs/starfish-rbc-protocol.md b/docs/starfish-rbc-protocol.md new file mode 100644 index 00000000..26cf7001 --- /dev/null +++ b/docs/starfish-rbc-protocol.md @@ -0,0 +1,647 @@ +# Starfish-RBC protocol specification + +Status: design milestone, not implemented + +This document specifies the first correctness-oriented prototype of Starfish with reliable header +certification and a signature-free MAC configuration. The provisional CLI name is `starfish-rbc`. + +The protocol is intentionally conservative. It reuses plain Starfish for transaction-data +availability, DAG ordering, and commitment, while adding a Bracha reliable-broadcast layer for +block headers. It uses the same validator, committee, networking, storage, orchestrator, and +benchmark setup as the other protocols in this repository. + +This document is not a claim that the protocol is already proved or implemented. The motivating +work-in-progress note gives reliable-delivery and MAC-vector ingredients, but does not prove their +composition with Starfish. The proof obligations below must be discharged before making a safety +or liveness claim. + +## 1. Goals + +The first prototype has five goals: + +1. Replace transferable block signatures with receiver-specific MAC authentication without + allowing a Byzantine author to make a block permanently acceptable to only part of the honest + committee. +2. Preserve `BlockReference.digest` as a hash of canonical block content only. Authentication + remains a sidecar and does not change block identity. +3. Certify only the Starfish header, including its transaction commitment. Transaction payload + availability remains the responsibility of Starfish's existing Reed-Solomon and acknowledgment + layer. +4. Keep the reliable-broadcast message flow identical while selecting Ed25519, ML-DSA-44, + ML-DSA-65, or a MAC vector for the author's initial header authentication. This gives an + apples-to-apples comparison. +5. Establish a correctness baseline before adding tree dissemination, fast paths, or early + uncertified consensus participation. + +The current `starfish-mac`, `starfish-speed-mac`, `sparse-starfish-speed-mac`, and +`bluestreak-mac` modes remain lower-bound benchmark experiments. Changing only their block +authentication does not turn them into complete Byzantine protocols. + +Conceptually, selection is represented as two independent fields rather than copied protocol +variants: + +```text +ProtocolConfig { + consensus: StarfishRbc, + initial_header_authentication: Ed25519 | MlDsa44 | MlDsa65 | Mac, +} +``` + +The consensus selector enables RBC and the clean-DAG rules. The authentication selector changes +only `HeaderProposal.initial_authentication`. + +## 2. Version-one scope + +Version one assumes: + +- a static committee for the duration of a run; +- Byzantine voting stake strictly below one third of total stake; +- reliable point-to-point communication after GST in the usual partially synchronous model; +- fixed pairwise MAC keys provisioned by the benchmark setup; +- honest validators remain online for the run; and +- no committee reconfiguration or epoch transition. + +Let `W` be total committee stake. Version one uses the repository's exact integer thresholds: + +```text +Q = floor(2W / 3) + 1 +V = floor(W / 3) + 1 +``` + +The transitions are `Q` ECHO stake to READY, `V` READY stake to READY, and `Q` READY stake to +delivery. For an equal-stake committee with `n = 3f + 1`, these are `2f + 1`, `f + 1`, and +`2f + 1`. Locally sent ECHO and READY actions count toward these thresholds. + +The following are out of scope for version one: + +- crash/restart safety and replay of locally observed phase evidence; +- production key establishment, rotation, or compromise recovery; +- BLS control certificates; +- Starfish-Speed, Sparse-Starfish-Speed, and Bluestreak integration; +- tree or bounded-fanout header dissemination; +- Sailfish's optimistic `VOTE`/fast-delivery path; +- the `DONE` reliable-delivery optimization; +- encoding ECHO and READY as DAG protocol blocks; and +- adversarial resource-exhaustion resistance beyond fixed message-size checks. + +These exclusions constrain the first implementation, not the eventual research direction. + +## 3. Protocol identity and authenticated statements + +### 3.1 Slot and value + +One reliable-broadcast instance exists for each slot: + +```text +Slot = (protocol_instance, committee_id, author, round) +``` + +The value proposed in a slot is a canonical Starfish header. Its identifier is the existing: + +```text +BlockReference = (author, round, content_digest) +``` + +`content_digest` commits to the existing canonical header content, including the transaction +commitment, parent references, acknowledgments, and other consensus-relevant fields. It does not +commit to the initial authentication sidecar or to reliable-broadcast messages. + +`protocol_instance` and `committee_id` are domain-separation inputs. They are not added to +`BlockReference`. Their exact derivation must be fixed before implementation; a committee/genesis +identifier is preferable to a human-readable CLI string. + +Header processing has three distinct gates: + +1. **Content validation** is deterministic and view-independent. It checks the canonical digest, + intrinsic field relationships, committee/round bounds, the committed transaction root, and + protocol syntax without requiring the local initial proof, transaction data, a shard, or locally + available dependencies. +2. **Initial authentication** determines only whether this validator may send ECHO for the + candidate. +3. **DAG admission and clean activation** resolve dependencies and determine whether the delivered + header can influence Starfish. + +The implementation must not keep these gates bundled in the current all-or-nothing block verifier. + +### 3.2 Initial header authentication + +The configured block-authentication method applies only to the author's initial header proof: + +- Ed25519 and ML-DSA sign a domain-separated statement containing the slot and content digest. +- In MAC mode, author `A` creates one receiver-specific tag for each validator `Q`: + +```text +InitialTag(A, Q, block_ref) = + MAC[k(A,Q)](INITIAL || Slot || block_ref || A || Q) +``` + +The ordered collection of these tags is the conceptual MAC vector. With direct dissemination, +each recipient receives only its own entry. The full vector is not transmitted to every peer. + +The local author does not need to authenticate its header to itself. Local construction is the +author's ECHO eligibility evidence; only messages sent to other validators need receiver-specific +initial tags. + +A valid initial proof permits an honest recipient to ECHO the header. It does not by itself make +the header clean, globally available, or safe to commit. + +### 3.3 Phase-message authentication + +ECHO and READY use pairwise MAC authentication for every initial header-authentication variant. +For phase sender `S` and recipient `Q`: + +```text +PhaseTag(S, Q, phase, block_ref) = + MAC[k(S,Q)](RBC_PHASE || Slot || phase || block_ref || S || Q) +``` + +The concrete encoding must use fixed-width, canonical fields and distinct domain values for +INITIAL, ECHO, and READY. The protocol must not rely on ambiguous string concatenation. + +An inbound phase message counts only if all of the following hold: + +- its recipient is the local validator; +- its sender is a known committee member; +- its claimed sender equals the peer on the direct connection; +- its pairwise MAC verifies for that sender and recipient; +- its slot fields agree with its block reference; and +- it belongs to the active protocol run and retained slot window. + +A forwarded or replayed phase message received from another peer never counts, even if its bytes +contain a valid tag addressed to the receiver. Local ECHO/READY actions are counted locally and do +not require a loopback network message. + +## 4. Messages + +Version one uses the following logical messages: + +```text +HeaderProposal { + slot, + canonical_header, + initial_authentication, +} + +Echo { + slot, + block_ref, + sender, + recipient, + phase_tag, +} + +Ready { + slot, + block_ref, + sender, + recipient, + phase_tag, +} + +HeaderRequest { + slot, + block_ref, +} + +HeaderResponse { + slot, + canonical_header, +} +``` + +ECHO and READY contain a block reference, not the header. This avoids rebroadcasting each header +quadratically. Header request/response traffic transports data only and is never counted as quorum +testimony. + +An honest ECHO or READY sender must possess the matching content-validated header. Consequently, +when a threshold is observed before the local header arrives, the receiver can request the header +from recorded direct ECHO or READY senders. A response is accepted only after recomputing the +content digest and validating the header's structure. + +`CanonicalHeader` excludes the initial-authentication sidecar. A `HeaderResponse` therefore neither +needs nor confers a valid local initial proof. It supplies the content bytes needed for a transition +that is authorized by the receiver's own directly authenticated RBC evidence. + +The RBC header-retrieval path never asserts transaction-payload availability. Shards and full +transaction data continue to use the existing Starfish paths. + +## 5. Local state + +State is keyed by slot, with evidence separated by candidate block reference: + +```text +SlotState { + echoed: Option, + readied: Option, + delivered: Option, + candidates: Map, +} + +CandidateState { + header: Option, + initial_authentication_valid: bool, + echo_senders: AuthoritySet, + ready_senders: AuthoritySet, + echo_quorum_observed: bool, + ready_validity_observed: bool, + ready_quorum_observed: bool, + dirty_admitted: bool, + rbc_delivered: bool, + clean: bool, +} +``` + +The `echoed`, `readied`, and `delivered` guards are slot-global, not per digest. Evidence is kept per +digest so that Byzantine equivocation can be observed without locking the receiver to the first +value it sees. + +In particular: + +- an honest validator sends at most one ECHO in a slot; +- an honest validator sends at most one READY in a slot; +- the READY value need not equal the value previously ECHOed, because a later quorum may select a + different value; and +- an honest validator delivers at most one value in a slot. + +For correctness in version one, slot-global locks, phase evidence, threshold latches, and headers +advertised by an honest local phase action are retained for the whole benchmark run. Unsupported +candidate bodies may be evicted because they can be fetched again by digest, but eviction must not +discard evidence or a pending transition. A proved retirement boundary and adversarial storage +bounds are deferred with crash/restart support; arbitrary cache eviction is not a protocol action. + +Sending a local phase is one atomic state transition: set the slot-global guard, insert the local +authority into that candidate's phase-sender set, and enqueue separately authenticated messages for +all other validators. Threshold checks include this local evidence. + +## 6. State machine + +### 6.1 Receiving an initial header + +On `HeaderProposal` for candidate `R`: + +1. Validate committee membership, slot consistency, header syntax, canonical content digest, + protocol-specific fields, and resource bounds. +2. Store the fixed-size-checked, content-valid header as a candidate even if its local initial + authenticator is missing or invalid. This permits later RBC delivery to repair a poisoned + recipient tag. +3. Verify that version one's proposal was received directly from the claimed author, then verify the + selected initial authentication method for the local recipient. For the local author's own + header, successful local construction supplies this evidence without a loopback signature or + MAC. +4. If the direct-author check and proof are valid and the slot-global ECHO guard is empty, record the + local ECHO immediately and send a recipient-specific ECHO for `R` to every other validator. + Header RBC does not wait for parents, acknowledgments, transaction data, or a shard to arrive. +5. Independently, connect the candidate to the dirty DAG through the normal dependency manager once + its dirty dependencies are present. This follows Sailfish's dirty/clean setup and does not make + the candidate consensus-visible. + +A relayed header may be retained as a content-valid candidate, but it does not trigger ECHO in +version one. Tree dissemination later changes this eligibility rule to accept a relayed +receiver-specific author proof and must extend the integrity argument accordingly. + +### 6.2 Receiving ECHO + +On a valid direct `Echo(R, S)`: + +1. Record `S` once in `R.echo_senders`. A Byzantine sender may appear in the evidence sets of + multiple conflicting candidates, but its stake counts only once per candidate. +2. When ECHO stake for `R` reaches `Q`, latch `echo_quorum_observed` and: + - if the header is absent, request it from multiple recorded ECHO senders; + - validate and store the returned header; and + - once the header is present, send READY for `R` if the slot-global READY guard is empty. + +An ECHO quorum never bypasses content validation. + +Each honest validator counts its own locally recorded ECHO, including when that validator is the +slot author. It also sends ECHO to every other validator. The implementation must not inherit +Sailfish++'s current exclusion of the block author from ECHO stake: at `n = 3f + 1`, excluding an +honest author's ECHO leaves only `2f` honest non-author ECHOs and lets `f` Byzantine validators stop +honest-author progress by withholding theirs. + +### 6.3 Receiving READY + +On a valid direct `Ready(R, S)`: + +1. Record `S` once in `R.ready_senders`. +2. When READY stake for `R` reaches `V`, latch `ready_validity_observed` and: + - if the header is absent, request it from multiple recorded READY senders; + - validate and store the returned header; and + - once the header is present, send READY for `R` if the slot-global READY guard is empty. +3. When READY stake for `R` reaches `Q`, latch `ready_quorum_observed`. Once the validated header is + present, locally deliver `R` if the slot-global delivery guard is empty. Emit a local + RBC-delivery event; do not construct a purportedly transferable certificate from the observed + pairwise MACs. + +READY amplification and delivery are independent of whether the local initial header proof was +valid. This is the mechanism that repairs selective or poisoned author-to-recipient MAC entries. + +### 6.4 Header retrieval + +Every honest ECHO and READY sender is a header holder. Retrieval therefore proceeds as follows: + +1. Select multiple direct phase senders as candidate sources. +2. Request the exact `BlockReference`. +3. Accept the first response whose recomputed digest and structural validation match. +4. Re-evaluate all latched ECHO/READY triggers immediately after storing the header. +5. Continue requesting while threshold progress is blocked and untried holders remain. + +At least one honest holder exists in every `V`-stake READY set. At least `f + 1` honest holders exist +in an equal-stake `2f + 1` ECHO quorum. Byzantine responses can delay retrieval but cannot change +the accepted content. An honest validator that sends ECHO or READY retains the canonical header +until the instance's state is safely retired; in version one, that means the end of the run. + +## 7. Dirty and clean DAG integration + +Starfish-RBC follows the dual dirty/clean organization already used by Sailfish++ and other +certified variants. + +The lifecycle predicates are distinct: + +```text +candidate = canonical header is content-valid +dirty = candidate is DAG-admitted but not yet clean +delivered = local READY evidence reached Q for the candidate +clean = delivered candidate has clean referenced dependencies +``` + +Transaction-data availability is a separate predicate throughout this lifecycle. + +### 7.1 Dirty state + +A locally authenticated, dependency-connected header may enter the dirty DAG before RBC delivery. +Dirty state may support: + +- candidate retention; +- parent fetching; +- header and shard synchronization; and +- RBC progress. + +Dirty state must not influence proposal eligibility, parent selection, Starfish votes, +acknowledgments, leader decisions, ordering, or commitment. + +### 7.2 Clean activation + +A header becomes clean only when: + +- the local RBC instance delivered that exact `BlockReference`; +- the header is present and content-valid; +- every direct parent is present and clean; and +- every referenced block whose acknowledgment could influence Starfish sequencing is present and + clean. + +RBC delivery and clean activation require neither transaction data nor a local shard. Here, clean +means that the canonical header is reliably delivered and dependency-closed. A Starfish +acknowledgment requires both `clean(header)` and `data_available(header)`. + +RBC delivery may therefore precede clean activation. A Byzantine delivered header with a dangling +dependency remains outside the clean DAG and cannot block progress by honest clean vertices. + +Local RBC delivery is an alternative admission authority to the author's initial proof. If the +local initial tag was invalid or missing, the delivered header is inserted through an +RBC-authorized path and may become clean after its dependencies do. No forwarded evidence bundle +can exercise this path; it requires the local slot state to have reached delivery. + +The generic dual-DAG rule that infers cleanliness from `f + 1` references in later rounds is +disabled for `starfish-rbc`. Only local RBC delivery can supply the certification predicate; +descendant references are neither a replacement certificate nor transferable evidence of the +direct phase messages observed elsewhere. + +### 7.3 Starfish-specific clean-only rules + +For `starfish-rbc`: + +- honest proposals select only clean parents; +- proposal-round advancement requires a clean quorum in the preceding round; +- a transaction-data acknowledgment is queued only after its target is both data-available and + clean; +- clean activation rechecks acknowledgment eligibility when transaction data arrived first; +- voting and certifying blocks counted by the Starfish committer must be clean; +- committed leaders must be clean; +- the linearizer follows only clean parent and acknowledgment references; and +- a block cannot be committed merely because it exists in the dirty DAG. + +The existing generic dual-DAG helpers are useful but do not by themselves enforce all of these +Starfish acknowledgment and linearizer rules. Each consumer must be audited explicitly. + +## 8. Initial dissemination + +Version one uses direct author-to-all dissemination: + +- Ed25519 and ML-DSA send the same public signature to every recipient. +- MAC mode sends each recipient only its own 32-byte initial tag. +- All modes then run exactly the same pairwise-MAC ECHO/READY protocol. + +Every ECHO and READY broadcast is materialized separately for each recipient because its phase tag +binds that recipient. A broadcaster must never clone one serialized, tagged phase message to all +peers. The sender records its own phase action locally instead of creating a loopback tag. + +This direct strategy is the correctness baseline. It deliberately separates reliable-delivery +correctness from routing failures. + +A later tree-dissemination milestone may send a relay the tags for its assigned subtree and let it +forward smaller sub-bundles. That optimization must include redundant paths or a timeout fallback; +a single tree containing a Byzantine relay is not live. Tree dissemination must not alter the RBC +state machine or its clean-DAG rules. + +Under the motivating paper's broad definition, an author-to-recipient MAC that remains verifiable +after a third party relays it is transferable authentication for that intended recipient, although +it is not a publicly verifiable signature. The future tree variant therefore lies outside the +paper's no-transferable-authentication lower bound and needs its own integrity argument. + +## 9. Safety argument to complete + +The implementation and accompanying proof must establish at least the following lemmas. + +### 9.1 Honest-author integrity + +If an honest validator ECHOs a header attributed to an honest author, the author created the +matching initial proof. An adversary cannot produce an honest recipient's valid initial MAC or a +valid public signature for a different header. + +The recipient of a pairwise MAC also knows its key and could fabricate a tag addressed to itself. +An honest recipient follows the protocol and never does so. Byzantine recipients control less than +one-third stake and therefore cannot create an ECHO quorum for a value that no honest recipient +authenticated. The later relayed-INIT variant must state this argument explicitly rather than +claiming public non-repudiation from a MAC. + +### 9.2 Unique ECHO quorum + +Two conflicting values cannot both obtain more than two-thirds ECHO stake. Their ECHO sets +intersect in more than one-third stake, which contains an honest validator; the slot-global ECHO +guard prevents that validator from ECHOing both. + +### 9.3 Unique READY value + +The first honest READY for a value is rooted in an ECHO quorum. Subsequent honest READY messages +are rooted either in that quorum or in more than one-third READY stake, which contains an honest +READY sender. Therefore honest READY propagation cannot originate independently for two values. + +### 9.4 Reliable-delivery agreement and totality + +If one honest validator delivers `R`, more than one-third honest stake sent READY for `R`. Reliable +direct delivery of those READY messages causes every honest validator to amplify READY and +eventually observe a quorum. Header-holder retrieval gives every honest validator the exact bytes +needed to deliver `R`. + +### 9.5 Starfish composition + +Only locally delivered, dependency-closed clean headers influence Starfish. Reliable delivery +provides a consistent value for each author/round slot and prevents false attribution to an honest +author. The Starfish safety argument must then be checked over the clean DAG, including its +acknowledgment-based linearizer. + +These lemmas are proof obligations. They are not established merely by reusing Sailfish++ code. + +## 10. Liveness argument to complete + +For an honest author after GST: + +1. Direct dissemination eventually gives every honest validator the header and a valid initial + proof. +2. Honest parent selection ensures the referenced parent closure is eventually clean everywhere. +3. All honest validators ECHO the same reference. +4. Every honest validator observes ECHO quorum, sends READY, observes READY quorum, and delivers. +5. The header becomes clean once its already-clean dependencies are locally present. +6. At least quorum honest stake can therefore produce clean vertices in every live round, allowing + plain Starfish to advance and commit. + +The proof must also show that dirty Byzantine candidates, missing initial tags, invalid phase MACs, +and dangling Byzantine dependencies cannot influence clean Starfish state or prevent an honestly +scheduled quorum from progressing. Fair processing despite Byzantine traffic is assumed; +resource-exhaustion resistance is outside the version-one model. + +## 11. Required adversarial tests + +Implementation begins with deterministic tests for the failure modes, not only happy-path smoke +tests. + +### 11.1 RBC unit tests + +- `n = 4, f = 1`: a Byzantine author gives candidate X to one honest validator and candidate Y to + two others. A validator that first saw X must still process quorum traffic for Y. +- A Byzantine author provides valid initial tags to enough validators to form an ECHO quorum but + gives D an invalid or missing tag; D does not ECHO but eventually retrieves and delivers the same + header through READY evidence. +- A Byzantine author distributes valid receiver tags for conflicting headers; at most one value is + delivered by honest validators. +- One sender's duplicate ECHO or READY counts once. +- The slot author's locally recorded ECHO and directly received ECHOs count toward the ECHO quorum; + excluding the author makes the `n = 3f + 1` honest-sender case non-live when all Byzantine + validators withhold. +- Byzantine senders may equivocate across values without making an honest sender violate its + slot-global guards. +- Invalid, wrong-recipient, wrong-phase, stale, and wrong-slot phase MACs are rejected. +- A valid phase message replayed through a different peer is not counted as direct testimony. +- ECHO or READY quorum without local header triggers retrieval and does not advance until a valid + matching header is present. +- A threshold reached before header retrieval remains latched and fires immediately after the + matching header is stored. +- A tampered `HeaderResponse` is rejected by content-digest validation. + +### 11.2 DAG integration tests + +- Locally authenticated but undelivered headers remain dirty. +- RBC-delivered headers with an unclean parent remain outside the clean DAG. +- Cleaning the parent activates the already-delivered child. +- Dirty headers cannot advance the proposal round or become proposal parents. +- Later-round references cannot infer that an RBC-undelivered header is clean. +- Data availability for an unclean target does not queue a Starfish acknowledgment. +- Starfish voting, leader selection, and linearization ignore dirty blocks. +- No honest node commits two values for one `(author, round)` slot. + +### 11.3 End-to-end tests + +- Four honest validators commit using Ed25519, ML-DSA-44, ML-DSA-65, and MAC initial + authentication with identical RBC message flow. +- A poisoned-recipient-tag run still lets every honest validator clean and commit the same header. +- Byzantine equivocation does not split committed histories. +- A Byzantine dangling-parent block does not prevent quorum honest progress. +- Header recovery succeeds when quorum evidence arrives before header data. +- Existing protocols retain their current behavior and test results. + +Crash/restart tests are intentionally deferred until sent-phase and delivered-slot state is made +durable. + +## 12. Benchmark plan + +The fair comparison holds Starfish-RBC ordering, reliable broadcast, direct dissemination, load, +committee, topology, and timeout configuration constant. Only initial header authentication varies: + +- `starfish-rbc` + Ed25519; +- `starfish-rbc` + ML-DSA-44; +- `starfish-rbc` + ML-DSA-65; and +- `starfish-rbc` + MAC. + +Native Starfish with Ed25519 and ML-DSA should also be measured separately. That comparison shows +the total cost of reliable delivery, but it must not be presented as an isolated authentication +comparison because the message flows differ. + +Metrics should separate: + +- initial header-authentication bytes and CPU; +- ECHO/READY phase-MAC bytes and CPU; +- header proposal, phase, and recovery traffic; +- time from dirty admission to RBC delivery and clean activation; +- block and transaction commit latency; +- throughput and per-node inbound/outbound bandwidth; and +- author egress versus aggregate network traffic. + +MAC vectors and public signatures both require each non-author to receive the header in an +all-honest direct run. The expected MAC benefit is proof size, computation, and later author/tree +egress relative to large post-quantum signatures, not a reduction in the number of required +recipients. + +Benchmark results should live in experiment artifacts or a concise PR graph/summary, not as +long-lived tables in the protocol documentation. + +## 13. Contained implementation boundary + +The implementation should add one `StarfishRbc` capability path and one isolated RBC service. It +may reuse networking, stake aggregation, dirty/clean storage, and event plumbing, but must not copy +the following Sailfish++ or legacy MAC behavior: + +- per-digest send/delivery guards or first-seen canonical locking; +- exclusion of the block author's ECHO, optimistic `VOTE`, or Sailfish timeout semantics; +- unsigned phase messages without recipient, session context, and a phase MAC; +- cloning one identical phase message to every recipient; +- full-vector direct-author transport when one recipient tag is sufficient; +- an all-or-nothing verifier that combines canonical content with initial authentication; or +- inferred-clean promotion and pre-clean acknowledgment queuing. + +Plain Starfish, Starfish-Speed, Sparse-Starfish-Speed, Bluestreak, Sailfish++, and their existing +authentication selections remain behaviorally unchanged. The first prototype does not create an +RBC copy of every protocol. + +## 14. Implementation milestones + +Each milestone is committed separately. + +1. **Specification:** this document, with no protocol code changes. +2. **RBC kernel:** domain-separated phase MACs, slot-global ECHO/READY state, per-value evidence, + header-holder tracking, and adversarial unit tests. +3. **Header staging and retrieval:** split content validation from initial authentication, add + fixed-size-checked candidate staging with pending triggers, and fetch headers from direct phase + senders. +4. **Certified Starfish integration:** add `starfish-rbc`, selectable initial authentication, + dirty/clean lifecycle, clean-only acknowledgments, and clean-only consensus/linearization. +5. **End-to-end validation:** poisoned-tag, equivocation, dangling-parent, and all-authentication + commit tests. +6. **Tree dissemination:** subtree tag bundles, redundant routing/fallback, and matching signature + baselines. +7. **Recovery:** durable phase locks and delivered state, evidence replay, late-node synchronization, + and restart tests. +8. **Benchmarks:** direct and tree comparisons with results reported outside this specification. + +## 15. Decisions still required before implementation + +The protocol behavior above is fixed, but implementation must still choose: + +- the exact `protocol_instance` and `committee_id` derivation; +- the canonical byte encoding for phase-MAC statements; +- a safe post-v1 state-retirement and garbage-collection rule; +- header-holder request fanout and retry timing; +- the final CLI spelling for MAC authentication; and +- whether legacy unsafe `*-mac` aliases are renamed or retained as lower-bound benchmarks. + +None of these choices may weaken the slot-global phase guards, direct-message checks, header +availability requirement, or clean-only Starfish boundary. From 514d0a055c19fb8fe344a2810abfc8a175c81378 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:43:10 +0200 Subject: [PATCH 22/62] Add Starfish-RBC kernel --- crates/starfish-core/src/crypto.rs | 10 + crates/starfish-core/src/lib.rs | 1 + crates/starfish-core/src/starfish_rbc.rs | 2070 ++++++++++++++++++++++ docs/starfish-rbc-protocol.md | 154 +- 4 files changed, 2189 insertions(+), 46 deletions(-) create mode 100644 crates/starfish-core/src/starfish_rbc.rs diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index ea73416c..df659c4f 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -502,6 +502,16 @@ impl MacKey { hasher.update(content_digest.as_ref()); MacTag(hasher.finalize().into()) } + + /// Authenticate an already-canonical Starfish-RBC statement. + /// + /// Statement construction and domain separation live in the RBC module; + /// keeping keyed-hasher access here avoids exposing the secret key bytes. + pub(crate) fn compute_rbc_tag(&self, statement: &[u8]) -> MacTag { + let mut hasher = Blake3Hasher::new_keyed(&self.0); + hasher.update(statement); + MacTag(hasher.finalize().into()) + } } /// Generate deterministic, symmetric pairwise keyrings for local benchmarks diff --git a/crates/starfish-core/src/lib.rs b/crates/starfish-core/src/lib.rs index ea6e6142..4c72bbe7 100644 --- a/crates/starfish-core/src/lib.rs +++ b/crates/starfish-core/src/lib.rs @@ -29,6 +29,7 @@ pub mod prometheus; mod rocks_store; mod runtime; pub mod shard_reconstructor; +mod starfish_rbc; mod stat; mod state; pub(crate) mod store; diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs new file mode 100644 index 00000000..d49ee2a3 --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -0,0 +1,2070 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Synchronous reliable-broadcast kernel for Starfish-RBC. +//! +//! This module is deliberately not connected to networking or DAG admission +//! yet. The next milestones will supply content-validated headers and expand +//! multicast effects into recipient-specific network messages. + +#![allow(dead_code)] + +use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; + +use ahash::AHashMap; +use serde::{Deserialize, Deserializer, Serialize, de}; + +use crate::{ + committee::{Committee, QuorumThreshold, StakeAggregator, ValidityThreshold}, + crypto::{Blake3Hasher, MacKey, MacTag}, + types::{ + AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockReference, + MAX_COMMITTEE_SIZE, RoundNumber, Stake, + }, +}; + +const PROTOCOL_DOMAIN: &[u8; 15] = b"STARFISH_RBC_V1"; +const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_V1_COMMITTEE_ID"; +const INITIAL_KIND: u8 = 0x00; +const ECHO_KIND: u8 = 0x01; +const READY_KIND: u8 = 0x02; + +const PROTOCOL_INSTANCE_SIZE: usize = 32; +const COMMITTEE_ID_SIZE: usize = 32; +const BLOCK_REFERENCE_SIZE: usize = 2 + 4 + 32; +const BASE_STATEMENT_SIZE: usize = PROTOCOL_DOMAIN.len() + + 1 + + 1 + + PROTOCOL_INSTANCE_SIZE + + COMMITTEE_ID_SIZE + + BLOCK_REFERENCE_SIZE; +const MAC_STATEMENT_SIZE: usize = BASE_STATEMENT_SIZE + 2 + 2; + +#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize)] +pub(crate) struct RbcProtocolInstanceId([u8; PROTOCOL_INSTANCE_SIZE]); + +impl RbcProtocolInstanceId { + pub(crate) fn new(bytes: [u8; PROTOCOL_INSTANCE_SIZE]) -> Result { + if bytes.iter().all(|byte| *byte == 0) { + return Err(RbcError::ZeroProtocolInstance); + } + Ok(Self(bytes)) + } +} + +impl<'de> Deserialize<'de> for RbcProtocolInstanceId { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let bytes = <[u8; PROTOCOL_INSTANCE_SIZE]>::deserialize(deserializer)?; + Self::new(bytes).map_err(de::Error::custom) + } +} + +impl fmt::Debug for RbcProtocolInstanceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "RbcInstance({})", hex::encode(&self.0[..4])) + } +} + +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub(crate) struct RbcCommitteeId([u8; COMMITTEE_ID_SIZE]); + +impl RbcCommitteeId { + fn derive(committee: &Committee) -> Result { + if committee.len() > MAX_COMMITTEE_SIZE as usize { + return Err(RbcError::CommitteeTooLarge(committee.len())); + } + let committee_size = u16::try_from(committee.len()) + .map_err(|_| RbcError::CommitteeTooLarge(committee.len()))?; + let info_length = u16::try_from(committee.info_length()) + .map_err(|_| RbcError::InvalidInfoLength(committee.info_length()))?; + + let mut hasher = Blake3Hasher::new_derive_key(COMMITTEE_ID_DERIVE_CONTEXT); + hasher.update(&committee_size.to_be_bytes()); + hasher.update(&committee.validity_threshold().to_be_bytes()); + hasher.update(&committee.quorum_threshold().to_be_bytes()); + hasher.update(&info_length.to_be_bytes()); + hasher.update(&committee.optimistic_fast_threshold().to_be_bytes()); + hasher.update(&committee.optimistic_vote_threshold().to_be_bytes()); + hasher.update(&committee.optimistic_ready_threshold().to_be_bytes()); + + for authority in committee.authorities() { + let stake = committee + .get_stake(authority) + .ok_or(RbcError::UnknownAuthority(authority))?; + let public_key = committee + .get_public_key(authority) + .ok_or(RbcError::UnknownAuthority(authority))?; + let bls_public_key = committee + .get_bls_public_key(authority) + .ok_or(RbcError::UnknownAuthority(authority))?; + let ml_dsa_44_public_key = committee + .get_ml_dsa_44_public_key(authority) + .ok_or(RbcError::UnknownAuthority(authority))?; + let ml_dsa_65_public_key = committee + .get_ml_dsa_65_public_key(authority) + .ok_or(RbcError::UnknownAuthority(authority))?; + + hasher.update(&authority.to_be_bytes()); + hasher.update(&stake.to_be_bytes()); + hasher.update(&public_key.to_bytes()); + hasher.update(&bls_public_key.to_bytes()); + hasher.update(&ml_dsa_44_public_key.to_bytes()); + hasher.update(&ml_dsa_65_public_key.to_bytes()); + } + + Ok(Self(hasher.finalize().into())) + } +} + +impl fmt::Debug for RbcCommitteeId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "RbcCommittee({})", hex::encode(&self.0[..4])) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct RbcContext { + protocol_instance: RbcProtocolInstanceId, + committee_id: RbcCommitteeId, + initial_authentication: BlockAuthenticationScheme, +} + +impl RbcContext { + fn new( + protocol_instance: RbcProtocolInstanceId, + committee: &Committee, + initial_authentication: BlockAuthenticationScheme, + ) -> Result { + validate_committee(committee)?; + Ok(Self { + protocol_instance, + committee_id: RbcCommitteeId::derive(committee)?, + initial_authentication, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum RbcPhase { + Echo, + Ready, +} + +impl RbcPhase { + fn statement_kind(self) -> u8 { + match self { + Self::Echo => ECHO_KIND, + Self::Ready => READY_KIND, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct RbcPhaseMessage { + block_ref: BlockReference, + sender: AuthorityIndex, + recipient: AuthorityIndex, + phase: RbcPhase, + tag: MacTag, +} + +impl RbcPhaseMessage { + pub(crate) fn block_ref(&self) -> BlockReference { + self.block_ref + } + + pub(crate) fn sender(&self) -> AuthorityIndex { + self.sender + } + + pub(crate) fn recipient(&self) -> AuthorityIndex { + self.recipient + } + + pub(crate) fn phase(&self) -> RbcPhase { + self.phase + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum RbcEffect { + /// The network adapter must specialize this intent for each recipient by + /// calling `make_phase_message`; no tagged message may be cloned. + MulticastPhase { + phase: RbcPhase, + block_ref: BlockReference, + }, + NeedHeader { + block_ref: BlockReference, + holders: AuthoritySet, + }, + Deliver(BlockReference), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum RbcError { + EmptyCommittee, + CommitteeTooLarge(usize), + InvalidInfoLength(usize), + InvalidCommitteeStake(AuthorityIndex), + TotalStakeOverflow, + InvalidValidityThreshold { + expected: Stake, + actual: Stake, + }, + InvalidQuorumThreshold { + expected: Stake, + actual: Stake, + }, + InvalidDerivedInfoLength { + expected: usize, + actual: usize, + }, + ZeroProtocolInstance, + InvalidKeyringLength { + expected: usize, + actual: usize, + }, + UnknownAuthority(AuthorityIndex), + GenesisSlot, + HeaderUnavailable(BlockReference), + WrongRecipient { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + SenderPeerMismatch { + sender: AuthorityIndex, + peer: AuthorityIndex, + }, + PhaseNotAuthorized { + phase: RbcPhase, + block_ref: BlockReference, + }, + LoopbackPhase, + InvalidPhaseTag, + InvalidInitialTag, + InitialSignatureRequiresSignatureAuthentication, + InitialMacRequiresMacAuthentication, + InitialAuthorMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, +} + +impl fmt::Display for RbcError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyCommittee => f.write_str("Starfish-RBC committee is empty"), + Self::CommitteeTooLarge(size) => { + write!(f, "Starfish-RBC committee is too large: {size}") + } + Self::InvalidInfoLength(length) => { + write!( + f, + "Starfish-RBC information length is out of range: {length}" + ) + } + Self::InvalidCommitteeStake(authority) => { + write!(f, "Starfish-RBC authority {authority} has invalid stake") + } + Self::TotalStakeOverflow => f.write_str("Starfish-RBC total stake overflow"), + Self::InvalidValidityThreshold { expected, actual } => write!( + f, + "Starfish-RBC validity threshold mismatch: expected {expected}, got {actual}" + ), + Self::InvalidQuorumThreshold { expected, actual } => write!( + f, + "Starfish-RBC quorum threshold mismatch: expected {expected}, got {actual}" + ), + Self::InvalidDerivedInfoLength { expected, actual } => write!( + f, + "Starfish-RBC information length mismatch: expected {expected}, got {actual}" + ), + Self::ZeroProtocolInstance => { + f.write_str("Starfish-RBC protocol instance must not be all zeroes") + } + Self::InvalidKeyringLength { expected, actual } => write!( + f, + "Starfish-RBC keyring length mismatch: expected {expected}, got {actual}" + ), + Self::UnknownAuthority(authority) => { + write!(f, "unknown Starfish-RBC authority {authority}") + } + Self::GenesisSlot => f.write_str("Starfish-RBC does not certify genesis slots"), + Self::HeaderUnavailable(block_ref) => { + write!( + f, + "Starfish-RBC header {block_ref} is not locally available" + ) + } + Self::WrongRecipient { expected, actual } => write!( + f, + "Starfish-RBC message recipient mismatch: expected {expected}, got {actual}" + ), + Self::SenderPeerMismatch { sender, peer } => write!( + f, + "Starfish-RBC message sender {sender} does not match direct peer {peer}" + ), + Self::PhaseNotAuthorized { phase, block_ref } => write!( + f, + "Starfish-RBC {phase:?} was not authorized for {block_ref}" + ), + Self::LoopbackPhase => { + f.write_str("Starfish-RBC loopback phase messages are not accepted") + } + Self::InvalidPhaseTag => f.write_str("Starfish-RBC phase MAC verification failed"), + Self::InvalidInitialTag => f.write_str("Starfish-RBC initial MAC verification failed"), + Self::InitialSignatureRequiresSignatureAuthentication => f.write_str( + "Starfish-RBC initial signature digest requires a signature authentication mode", + ), + Self::InitialMacRequiresMacAuthentication => { + f.write_str("Starfish-RBC initial MAC requires MAC authentication mode") + } + Self::InitialAuthorMismatch { expected, actual } => write!( + f, + "Starfish-RBC initial author mismatch: expected {expected}, got {actual}" + ), + } + } +} + +impl Error for RbcError {} + +struct CandidateState { + header_available: bool, + echoes: StakeAggregator, + readies: StakeAggregator, + echo_quorum_observed: bool, + ready_validity_observed: bool, + ready_quorum_observed: bool, + header_request_holders: AuthoritySet, +} + +impl CandidateState { + fn new() -> Self { + Self { + header_available: false, + echoes: StakeAggregator::new(), + readies: StakeAggregator::new(), + echo_quorum_observed: false, + ready_validity_observed: false, + ready_quorum_observed: false, + header_request_holders: AuthoritySet::default(), + } + } + + fn latch_thresholds(&mut self, validity_threshold: Stake, quorum_threshold: Stake) { + self.echo_quorum_observed |= self.echoes.get_stake() >= quorum_threshold; + self.ready_validity_observed |= self.readies.get_stake() >= validity_threshold; + self.ready_quorum_observed |= self.readies.get_stake() >= quorum_threshold; + } + + fn holders(&self) -> AuthoritySet { + self.echoes.votes | self.readies.votes + } +} + +#[derive(Default)] +struct SlotState { + echoed: Option, + readied: Option, + delivered: Option, + candidates: AHashMap, +} + +enum ProgressAction { + NeedHeader(AuthoritySet), + SendReady, + Deliver, + None, +} + +pub(crate) struct StarfishRbcKernel { + committee: Arc, + own_authority: AuthorityIndex, + context: RbcContext, + mac_keys: Arc>, + slots: BTreeMap>, +} + +impl StarfishRbcKernel { + pub(crate) fn new( + committee: Arc, + own_authority: AuthorityIndex, + protocol_instance: RbcProtocolInstanceId, + initial_authentication: BlockAuthenticationScheme, + mac_keys: Arc>, + ) -> Result { + let context = RbcContext::new(protocol_instance, &committee, initial_authentication)?; + if !committee.known_authority(own_authority) { + return Err(RbcError::UnknownAuthority(own_authority)); + } + if mac_keys.len() != committee.len() { + return Err(RbcError::InvalidKeyringLength { + expected: committee.len(), + actual: mac_keys.len(), + }); + } + Ok(Self { + committee, + own_authority, + context, + mac_keys, + slots: BTreeMap::new(), + }) + } + + pub(crate) fn context(&self) -> RbcContext { + self.context + } + + /// Record deterministic content validation and local header availability. + /// This does not authorize ECHO and does not imply initial authentication. + /// It remains module-private until header staging can supply a typed, + /// pinned content-validation result. + fn note_header_available( + &mut self, + block_ref: BlockReference, + ) -> Result, RbcError> { + self.validate_block_ref(&block_ref)?; + self.candidate_mut(block_ref).header_available = true; + Ok(self.drive(block_ref)) + } + + /// Authorize the one local ECHO for this slot. It remains module-private + /// until the integration layer can pass a typed direct-author proof (or a + /// local-creation capability) instead of relying on call ordering. + fn authorize_echo(&mut self, block_ref: BlockReference) -> Result, RbcError> { + self.validate_block_ref(&block_ref)?; + let header_available = self + .candidate(&block_ref) + .is_some_and(|candidate| candidate.header_available); + if !header_available { + return Err(RbcError::HeaderUnavailable(block_ref)); + } + + let own_authority = self.own_authority; + let committee = Arc::clone(&self.committee); + let slot = self.slot_mut(block_ref); + if slot.echoed.is_some() { + return Ok(Vec::new()); + } + slot.echoed = Some(block_ref); + slot.candidates + .entry(block_ref) + .or_insert_with(CandidateState::new) + .echoes + .add(own_authority, &committee); + + let mut effects = vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, + }]; + effects.extend(self.drive(block_ref)); + Ok(effects) + } + + pub(crate) fn handle_phase( + &mut self, + direct_peer: AuthorityIndex, + message: RbcPhaseMessage, + ) -> Result, RbcError> { + self.verify_phase_message(direct_peer, &message)?; + let committee = Arc::clone(&self.committee); + let candidate = self.candidate_mut(message.block_ref); + match message.phase { + RbcPhase::Echo => { + candidate.echoes.add(message.sender, &committee); + } + RbcPhase::Ready => { + candidate.readies.add(message.sender, &committee); + } + } + Ok(self.drive(message.block_ref)) + } + + /// Materialize one recipient-specific message for an untagged multicast + /// effect. The network adapter calls this once per non-local recipient. + pub(crate) fn make_phase_message( + &self, + phase: RbcPhase, + block_ref: BlockReference, + recipient: AuthorityIndex, + ) -> Result { + self.validate_block_ref(&block_ref)?; + let authorized = self.slot(&block_ref).is_some_and(|slot| { + let phase_authorized = match phase { + RbcPhase::Echo => slot.echoed == Some(block_ref), + RbcPhase::Ready => slot.readied == Some(block_ref), + }; + phase_authorized + && slot + .candidates + .get(&block_ref) + .is_some_and(|candidate| candidate.header_available) + }); + if !authorized { + return Err(RbcError::PhaseNotAuthorized { phase, block_ref }); + } + if !self.committee.known_authority(recipient) { + return Err(RbcError::UnknownAuthority(recipient)); + } + if recipient == self.own_authority { + return Err(RbcError::LoopbackPhase); + } + let statement = encode_mac_statement( + &self.context, + phase.statement_kind(), + &block_ref, + self.own_authority, + recipient, + ); + let tag = self.mac_keys[recipient as usize].compute_rbc_tag(&statement); + Ok(RbcPhaseMessage { + block_ref, + sender: self.own_authority, + recipient, + phase, + tag, + }) + } + + /// Produce the common 32-byte digest signed by Ed25519 or ML-DSA for an + /// initial Starfish-RBC header proposal. + pub(crate) fn initial_signature_digest( + &self, + block_ref: BlockReference, + ) -> Result<[u8; 32], RbcError> { + if self.context.initial_authentication == BlockAuthenticationScheme::MacVector { + return Err(RbcError::InitialSignatureRequiresSignatureAuthentication); + } + self.validate_block_ref(&block_ref)?; + let statement = encode_base_statement(&self.context, INITIAL_KIND, &block_ref); + Ok(blake3::hash(&statement).into()) + } + + /// Produce one receiver-specific initial MAC. The local author calls this + /// separately for every non-local recipient. + pub(crate) fn make_initial_mac_tag( + &self, + block_ref: BlockReference, + recipient: AuthorityIndex, + ) -> Result { + if self.context.initial_authentication != BlockAuthenticationScheme::MacVector { + return Err(RbcError::InitialMacRequiresMacAuthentication); + } + self.validate_block_ref(&block_ref)?; + if block_ref.authority != self.own_authority { + return Err(RbcError::InitialAuthorMismatch { + expected: block_ref.authority, + actual: self.own_authority, + }); + } + if !self.committee.known_authority(recipient) { + return Err(RbcError::UnknownAuthority(recipient)); + } + if recipient == self.own_authority { + return Err(RbcError::LoopbackPhase); + } + let statement = encode_mac_statement( + &self.context, + INITIAL_KIND, + &block_ref, + self.own_authority, + recipient, + ); + Ok(self.mac_keys[recipient as usize].compute_rbc_tag(&statement)) + } + + /// Verify the local receiver's initial MAC from the direct block author. + pub(crate) fn verify_initial_mac_tag( + &self, + direct_peer: AuthorityIndex, + block_ref: BlockReference, + tag: &MacTag, + ) -> Result<(), RbcError> { + if self.context.initial_authentication != BlockAuthenticationScheme::MacVector { + return Err(RbcError::InitialMacRequiresMacAuthentication); + } + self.validate_block_ref(&block_ref)?; + if direct_peer != block_ref.authority { + return Err(RbcError::InitialAuthorMismatch { + expected: block_ref.authority, + actual: direct_peer, + }); + } + if direct_peer == self.own_authority { + return Err(RbcError::LoopbackPhase); + } + let statement = encode_mac_statement( + &self.context, + INITIAL_KIND, + &block_ref, + direct_peer, + self.own_authority, + ); + let expected = self.mac_keys[direct_peer as usize].compute_rbc_tag(&statement); + if expected != *tag { + return Err(RbcError::InvalidInitialTag); + } + Ok(()) + } + + pub(crate) fn header_holders(&self, block_ref: &BlockReference) -> AuthoritySet { + self.candidate(block_ref) + .map(CandidateState::holders) + .unwrap_or_default() + } + + /// Recreate the current fetch effect for a durable retry timer. The first + /// `NeedHeader` effect is only a wake-up; recovery must retry until the + /// content-validated header becomes locally pinned. + pub(crate) fn retry_header_request( + &self, + block_ref: BlockReference, + ) -> Result, RbcError> { + self.validate_block_ref(&block_ref)?; + let Some(slot) = self.slot(&block_ref) else { + return Ok(None); + }; + let Some(candidate) = slot.candidates.get(&block_ref) else { + return Ok(None); + }; + let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; + let blocked_on_header = !candidate.header_available + && ((slot.readied.is_none() && ready_trigger) + || (slot.delivered.is_none() && candidate.ready_quorum_observed)); + Ok(blocked_on_header.then(|| RbcEffect::NeedHeader { + block_ref, + holders: candidate.holders(), + })) + } + + fn verify_phase_message( + &self, + direct_peer: AuthorityIndex, + message: &RbcPhaseMessage, + ) -> Result<(), RbcError> { + self.validate_block_ref(&message.block_ref)?; + if !self.committee.known_authority(direct_peer) { + return Err(RbcError::UnknownAuthority(direct_peer)); + } + if !self.committee.known_authority(message.sender) { + return Err(RbcError::UnknownAuthority(message.sender)); + } + if !self.committee.known_authority(message.recipient) { + return Err(RbcError::UnknownAuthority(message.recipient)); + } + if message.recipient != self.own_authority { + return Err(RbcError::WrongRecipient { + expected: self.own_authority, + actual: message.recipient, + }); + } + if message.sender != direct_peer { + return Err(RbcError::SenderPeerMismatch { + sender: message.sender, + peer: direct_peer, + }); + } + if message.sender == message.recipient { + return Err(RbcError::LoopbackPhase); + } + let statement = encode_mac_statement( + &self.context, + message.phase.statement_kind(), + &message.block_ref, + message.sender, + message.recipient, + ); + let expected = self.mac_keys[message.sender as usize].compute_rbc_tag(&statement); + if expected != message.tag { + return Err(RbcError::InvalidPhaseTag); + } + Ok(()) + } + + fn validate_block_ref(&self, block_ref: &BlockReference) -> Result<(), RbcError> { + if block_ref.round == 0 { + return Err(RbcError::GenesisSlot); + } + if !self.committee.known_authority(block_ref.authority) { + return Err(RbcError::UnknownAuthority(block_ref.authority)); + } + Ok(()) + } + + fn slot_mut(&mut self, block_ref: BlockReference) -> &mut SlotState { + self.slots + .entry(block_ref.round) + .or_default() + .entry(block_ref.authority) + .or_default() + } + + fn slot(&self, block_ref: &BlockReference) -> Option<&SlotState> { + self.slots + .get(&block_ref.round) + .and_then(|round| round.get(&block_ref.authority)) + } + + fn candidate_mut(&mut self, block_ref: BlockReference) -> &mut CandidateState { + self.slot_mut(block_ref) + .candidates + .entry(block_ref) + .or_insert_with(CandidateState::new) + } + + fn candidate(&self, block_ref: &BlockReference) -> Option<&CandidateState> { + self.slot(block_ref) + .and_then(|slot| slot.candidates.get(block_ref)) + } + + fn drive(&mut self, block_ref: BlockReference) -> Vec { + let validity_threshold = self.committee.validity_threshold(); + let quorum_threshold = self.committee.quorum_threshold(); + let mut effects = Vec::new(); + + loop { + let action = { + let slot = self.slot_mut(block_ref); + let can_send_ready = slot.readied.is_none(); + let can_deliver = slot.delivered.is_none(); + let candidate = slot + .candidates + .entry(block_ref) + .or_insert_with(CandidateState::new); + candidate.latch_thresholds(validity_threshold, quorum_threshold); + + let ready_trigger = + candidate.echo_quorum_observed || candidate.ready_validity_observed; + let blocked_on_header = !candidate.header_available + && ((can_send_ready && ready_trigger) + || (can_deliver && candidate.ready_quorum_observed)); + let holders = candidate.holders(); + if blocked_on_header && holders != candidate.header_request_holders { + candidate.header_request_holders = holders; + ProgressAction::NeedHeader(holders) + } else if candidate.header_available && can_send_ready && ready_trigger { + ProgressAction::SendReady + } else if candidate.header_available + && can_deliver + && candidate.ready_quorum_observed + { + ProgressAction::Deliver + } else { + ProgressAction::None + } + }; + + match action { + ProgressAction::NeedHeader(holders) => { + effects.push(RbcEffect::NeedHeader { block_ref, holders }); + break; + } + ProgressAction::SendReady => { + let own_authority = self.own_authority; + let committee = Arc::clone(&self.committee); + let slot = self.slot_mut(block_ref); + if slot.readied.is_none() { + slot.readied = Some(block_ref); + slot.candidates + .entry(block_ref) + .or_insert_with(CandidateState::new) + .readies + .add(own_authority, &committee); + effects.push(RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + }); + } + } + ProgressAction::Deliver => { + let slot = self.slot_mut(block_ref); + if slot.delivered.is_none() { + slot.delivered = Some(block_ref); + effects.push(RbcEffect::Deliver(block_ref)); + } + } + ProgressAction::None => break, + } + } + + effects + } +} + +fn validate_committee(committee: &Committee) -> Result<(), RbcError> { + if committee.is_empty() { + return Err(RbcError::EmptyCommittee); + } + if committee.len() > MAX_COMMITTEE_SIZE as usize { + return Err(RbcError::CommitteeTooLarge(committee.len())); + } + let mut total_stake = 0u64; + for authority in committee.authorities() { + let stake = committee + .get_stake(authority) + .ok_or(RbcError::UnknownAuthority(authority))?; + if stake == 0 { + return Err(RbcError::InvalidCommitteeStake(authority)); + } + total_stake = total_stake + .checked_add(stake) + .ok_or(RbcError::TotalStakeOverflow)?; + } + let expected_validity = total_stake / 3 + 1; + let expected_quorum = total_stake + .checked_mul(2) + .ok_or(RbcError::TotalStakeOverflow)? + / 3 + + 1; + if committee.validity_threshold() != expected_validity { + return Err(RbcError::InvalidValidityThreshold { + expected: expected_validity, + actual: committee.validity_threshold(), + }); + } + if committee.quorum_threshold() != expected_quorum { + return Err(RbcError::InvalidQuorumThreshold { + expected: expected_quorum, + actual: committee.quorum_threshold(), + }); + } + let committee_size = committee.len(); + let f = (committee_size - 1) / 3; + let expected_info_length = match committee_size % 3 { + 0 => f + 3, + 1 => f + 1, + _ => f + 2, + }; + if committee.info_length() != expected_info_length { + return Err(RbcError::InvalidDerivedInfoLength { + expected: expected_info_length, + actual: committee.info_length(), + }); + } + Ok(()) +} + +fn authentication_code(authentication: BlockAuthenticationScheme) -> u8 { + match authentication { + BlockAuthenticationScheme::Ed25519 => 0x00, + BlockAuthenticationScheme::MlDsa44 => 0x01, + BlockAuthenticationScheme::MlDsa65 => 0x02, + BlockAuthenticationScheme::MacVector => 0x03, + } +} + +fn encode_base_statement( + context: &RbcContext, + kind: u8, + block_ref: &BlockReference, +) -> [u8; BASE_STATEMENT_SIZE] { + let mut statement = [0u8; BASE_STATEMENT_SIZE]; + statement[..15].copy_from_slice(PROTOCOL_DOMAIN); + statement[15] = kind; + statement[16] = authentication_code(context.initial_authentication); + statement[17..49].copy_from_slice(&context.protocol_instance.0); + statement[49..81].copy_from_slice(&context.committee_id.0); + statement[81..83].copy_from_slice(&block_ref.authority.to_be_bytes()); + statement[83..87].copy_from_slice(&block_ref.round.to_be_bytes()); + statement[87..119].copy_from_slice(block_ref.digest.as_ref()); + statement +} + +fn encode_mac_statement( + context: &RbcContext, + kind: u8, + block_ref: &BlockReference, + sender: AuthorityIndex, + recipient: AuthorityIndex, +) -> [u8; MAC_STATEMENT_SIZE] { + let mut statement = [0u8; MAC_STATEMENT_SIZE]; + statement[..BASE_STATEMENT_SIZE] + .copy_from_slice(&encode_base_statement(context, kind, block_ref)); + statement[119..121].copy_from_slice(&sender.to_be_bytes()); + statement[121..123].copy_from_slice(&recipient.to_be_bytes()); + statement +} + +#[cfg(test)] +mod tests { + use std::collections::{HashSet, VecDeque}; + + use super::*; + use crate::{ + crypto::mac_keyrings_for_test, + types::{BlockDigest, BlockReference}, + }; + + const TEST_INSTANCE_BYTE: u8 = 0xA5; + + fn block(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { + BlockReference { + authority, + round, + digest: BlockDigest::from([marker; 32]), + } + } + + fn instance(marker: u8) -> RbcProtocolInstanceId { + RbcProtocolInstanceId::new([marker; 32]).unwrap() + } + + fn kernel( + committee: Arc, + keyrings: &[Vec], + own_authority: AuthorityIndex, + authentication: BlockAuthenticationScheme, + ) -> StarfishRbcKernel { + kernel_with_instance( + committee, + keyrings, + own_authority, + authentication, + TEST_INSTANCE_BYTE, + ) + } + + fn kernel_with_instance( + committee: Arc, + keyrings: &[Vec], + own_authority: AuthorityIndex, + authentication: BlockAuthenticationScheme, + instance_byte: u8, + ) -> StarfishRbcKernel { + StarfishRbcKernel::new( + committee, + own_authority, + instance(instance_byte), + authentication, + Arc::new(keyrings[own_authority as usize].clone()), + ) + .unwrap() + } + + fn phase_message( + committee: Arc, + keyrings: &[Vec], + sender: AuthorityIndex, + recipient: AuthorityIndex, + phase: RbcPhase, + block_ref: BlockReference, + ) -> RbcPhaseMessage { + let context = RbcContext::new( + instance(TEST_INSTANCE_BYTE), + committee.as_ref(), + BlockAuthenticationScheme::Ed25519, + ) + .unwrap(); + let statement = encode_mac_statement( + &context, + phase.statement_kind(), + &block_ref, + sender, + recipient, + ); + RbcPhaseMessage { + block_ref, + sender, + recipient, + phase, + tag: keyrings[sender as usize][recipient as usize].compute_rbc_tag(&statement), + } + } + + fn authorize_echo(kernel: &mut StarfishRbcKernel, block_ref: BlockReference) { + kernel.note_header_available(block_ref).unwrap(); + assert!(matches!( + kernel.authorize_echo(block_ref).unwrap().as_slice(), + [RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + .. + }] + )); + } + + fn holder_vec(holders: AuthoritySet) -> Vec { + holders.present().collect() + } + + fn pump_phase_effects( + kernels: &mut [StarfishRbcKernel], + initial_effects: Vec<(AuthorityIndex, Vec)>, + fetch_missing_headers: bool, + ) -> Vec> { + let mut queue: VecDeque<_> = initial_effects + .into_iter() + .flat_map(|(authority, effects)| { + effects.into_iter().map(move |effect| (authority, effect)) + }) + .collect(); + let mut deliveries = vec![Vec::new(); kernels.len()]; + + while let Some((owner, effect)) = queue.pop_front() { + match effect { + RbcEffect::MulticastPhase { phase, block_ref } => { + let messages: Vec<_> = (0..kernels.len()) + .filter(|recipient| *recipient != owner as usize) + .map(|recipient| { + let recipient = recipient as AuthorityIndex; + ( + recipient, + kernels[owner as usize] + .make_phase_message(phase, block_ref, recipient) + .unwrap(), + ) + }) + .collect(); + for (recipient, message) in messages { + let effects = kernels[recipient as usize] + .handle_phase(owner, message) + .unwrap(); + queue.extend(effects.into_iter().map(|effect| (recipient, effect))); + } + } + RbcEffect::NeedHeader { block_ref, .. } if fetch_missing_headers => { + let effects = kernels[owner as usize] + .note_header_available(block_ref) + .unwrap(); + queue.extend(effects.into_iter().map(|effect| (owner, effect))); + } + RbcEffect::NeedHeader { .. } => {} + RbcEffect::Deliver(block_ref) => { + deliveries[owner as usize].push(block_ref); + } + } + } + deliveries + } + + #[test] + fn canonical_statement_layout_is_fixed_width_and_big_endian() { + let context = RbcContext { + protocol_instance: RbcProtocolInstanceId([0x11; 32]), + committee_id: RbcCommitteeId([0x22; 32]), + initial_authentication: BlockAuthenticationScheme::MlDsa65, + }; + let block_ref = BlockReference { + authority: 0x0102, + round: 0x0304_0506, + digest: BlockDigest::from([0x33; 32]), + }; + let statement = encode_mac_statement(&context, ECHO_KIND, &block_ref, 0x0708, 0x090A); + + assert_eq!(statement.len(), 123); + assert_eq!(&statement[..15], PROTOCOL_DOMAIN); + assert_eq!(statement[15], ECHO_KIND); + assert_eq!(statement[16], 0x02); + assert_eq!(&statement[17..49], &[0x11; 32]); + assert_eq!(&statement[49..81], &[0x22; 32]); + assert_eq!(&statement[81..83], &[0x01, 0x02]); + assert_eq!(&statement[83..87], &[0x03, 0x04, 0x05, 0x06]); + assert_eq!(&statement[87..119], &[0x33; 32]); + assert_eq!(&statement[119..121], &[0x07, 0x08]); + assert_eq!(&statement[121..123], &[0x09, 0x0A]); + } + + #[test] + fn committee_id_is_stable_and_configuration_sensitive() { + let committee = Committee::new_test(vec![1, 2, 3, 4]); + let same_committee: Committee = + serde_yaml::from_str(&serde_yaml::to_string(&*committee).unwrap()).unwrap(); + let changed_stake = Committee::new_test(vec![1, 2, 3, 5]); + + let id = RbcCommitteeId::derive(&committee).unwrap(); + assert_eq!( + hex::encode(id.0), + "b64d92de81940c0965e6b8abc7b0b8ff409e3399343e6206d26be20f768fe970" + ); + assert_eq!(id, RbcCommitteeId::derive(&same_committee).unwrap()); + assert_ne!(id, RbcCommitteeId::derive(&changed_stake).unwrap()); + } + + #[test] + fn committee_id_encodes_authority_indices_above_255() { + let committee = Committee::new_test(vec![1; 300]); + let id = RbcCommitteeId::derive(&committee).unwrap(); + let round_trip: Committee = + serde_yaml::from_str(&serde_yaml::to_string(&*committee).unwrap()).unwrap(); + + assert_eq!(id, RbcCommitteeId::derive(&round_trip).unwrap()); + assert_ne!( + id, + RbcCommitteeId::derive(&Committee::new_test(vec![1; 299])).unwrap() + ); + } + + #[test] + fn oversized_deserialized_committee_is_rejected_before_authority_set_use() { + let committee = Committee::new_test(vec![1; MAX_COMMITTEE_SIZE as usize]); + let mut value = serde_yaml::to_value(committee.as_ref()).unwrap(); + let authorities = value + .as_mapping_mut() + .unwrap() + .get_mut(serde_yaml::Value::String("authorities".to_owned())) + .unwrap() + .as_sequence_mut() + .unwrap(); + authorities.push(authorities[0].clone()); + let oversized = Arc::new(serde_yaml::from_value::(value).unwrap()); + + assert_eq!(oversized.len(), MAX_COMMITTEE_SIZE as usize + 1); + let error = StarfishRbcKernel::new( + oversized, + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::Ed25519, + Arc::new(Vec::new()), + ) + .err() + .unwrap(); + assert_eq!( + error, + RbcError::CommitteeTooLarge(MAX_COMMITTEE_SIZE as usize + 1) + ); + } + + #[test] + fn inconsistent_deserialized_threshold_is_rejected_and_changes_id() { + let committee = Committee::new_test(vec![1, 1, 1, 1]); + let yaml = serde_yaml::to_string(&*committee).unwrap(); + let tampered_yaml = yaml.replacen("validity_threshold: 1", "validity_threshold: 2", 1); + assert_ne!(yaml, tampered_yaml); + let tampered: Committee = serde_yaml::from_str(&tampered_yaml).unwrap(); + + assert_ne!( + RbcCommitteeId::derive(&committee).unwrap(), + RbcCommitteeId::derive(&tampered).unwrap() + ); + let error = StarfishRbcKernel::new( + Arc::new(tampered), + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::Ed25519, + Arc::new(mac_keyrings_for_test(4)[0].clone()), + ) + .err() + .unwrap(); + assert!(matches!(error, RbcError::InvalidValidityThreshold { .. })); + } + + #[test] + fn protocol_instance_and_keyring_are_validated() { + assert_eq!( + RbcProtocolInstanceId::new([0; 32]), + Err(RbcError::ZeroProtocolInstance) + ); + let encoded = bincode::serialize(&[0u8; PROTOCOL_INSTANCE_SIZE]).unwrap(); + assert!(bincode::deserialize::(&encoded).is_err()); + let expected = instance(TEST_INSTANCE_BYTE); + assert_eq!( + bincode::deserialize::(&bincode::serialize(&expected).unwrap()) + .unwrap(), + expected + ); + let committee = Committee::new_test(vec![1; 4]); + let error = StarfishRbcKernel::new( + committee, + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::Ed25519, + Arc::new(Vec::new()), + ) + .err() + .unwrap(); + assert_eq!( + error, + RbcError::InvalidKeyringLength { + expected: 4, + actual: 0, + } + ); + } + + #[test] + fn initial_mac_is_recipient_specific_and_direct_author_bound() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let author = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::MacVector, + ); + let recipient = kernel( + Arc::clone(&committee), + &keyrings, + 1, + BlockAuthenticationScheme::MacVector, + ); + let other_recipient = kernel( + Arc::clone(&committee), + &keyrings, + 2, + BlockAuthenticationScheme::MacVector, + ); + let block_ref = block(0, 7, 0x44); + let tag = author.make_initial_mac_tag(block_ref, 1).unwrap(); + + recipient + .verify_initial_mac_tag(0, block_ref, &tag) + .unwrap(); + assert_eq!( + other_recipient.verify_initial_mac_tag(0, block_ref, &tag), + Err(RbcError::InvalidInitialTag) + ); + assert!(matches!( + recipient.verify_initial_mac_tag(2, block_ref, &tag), + Err(RbcError::InitialAuthorMismatch { .. }) + )); + } + + #[test] + fn signature_digest_binds_context_scheme_and_reference() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let ed25519 = kernel_with_instance( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + 0x11, + ); + let ml_dsa = kernel_with_instance( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::MlDsa44, + 0x11, + ); + let other_instance = kernel_with_instance( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + 0x22, + ); + let mac = kernel( + committee, + &keyrings, + 0, + BlockAuthenticationScheme::MacVector, + ); + let first = block(0, 9, 0x10); + let second = block(0, 9, 0x11); + + assert_ne!( + ed25519.initial_signature_digest(first).unwrap(), + ed25519.initial_signature_digest(second).unwrap() + ); + assert_ne!( + ed25519.initial_signature_digest(first).unwrap(), + ml_dsa.initial_signature_digest(first).unwrap() + ); + assert_ne!( + ed25519.initial_signature_digest(first).unwrap(), + other_instance.initial_signature_digest(first).unwrap() + ); + assert_eq!( + mac.initial_signature_digest(first), + Err(RbcError::InitialSignatureRequiresSignatureAuthentication) + ); + } + + #[test] + fn phase_messages_are_specialized_for_each_recipient() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut sender = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + let block_ref = block(0, 3, 0x21); + authorize_echo(&mut sender, block_ref); + let messages: Vec<_> = (1..4) + .map(|recipient| { + sender + .make_phase_message(RbcPhase::Echo, block_ref, recipient) + .unwrap() + }) + .collect(); + assert_eq!( + hex::encode(messages[0].tag.as_ref()), + "4a7a795ced01bb5fa2856de4b6664947e6564c8bc250c81a3073735346febf58" + ); + assert_eq!( + hex::encode(bincode::serialize(&messages[0]).unwrap()), + concat!( + "030000000000200000000000000021212121212121212121212121212121212121212121", + "21212121212121212121000001000000000020000000000000004a7a795ced01bb5fa2", + "856de4b6664947e6564c8bc250c81a3073735346febf58" + ) + ); + let distinct_tags: HashSet<_> = messages.iter().map(|message| message.tag).collect(); + assert_eq!(distinct_tags.len(), 3); + + for message in messages { + let recipient = message.recipient; + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + recipient, + BlockAuthenticationScheme::Ed25519, + ); + assert!(receiver.handle_phase(0, message).is_ok()); + } + } + + #[test] + fn phase_message_materialization_requires_authorized_local_state() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let first = block(0, 3, 0x23); + let conflicting = block(0, 3, 0x24); + let mut sender = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + + assert!(matches!( + sender.make_phase_message(RbcPhase::Echo, first, 1), + Err(RbcError::PhaseNotAuthorized { .. }) + )); + sender.note_header_available(first).unwrap(); + assert!(matches!( + sender.make_phase_message(RbcPhase::Echo, first, 1), + Err(RbcError::PhaseNotAuthorized { .. }) + )); + + assert!(matches!( + sender.authorize_echo(first).unwrap().as_slice(), + [RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + .. + }] + )); + let first_message = sender.make_phase_message(RbcPhase::Echo, first, 1).unwrap(); + assert_eq!( + first_message, + sender.make_phase_message(RbcPhase::Echo, first, 1).unwrap() + ); + + sender.note_header_available(conflicting).unwrap(); + assert!(sender.authorize_echo(conflicting).unwrap().is_empty()); + assert!(matches!( + sender.make_phase_message(RbcPhase::Echo, conflicting, 1), + Err(RbcError::PhaseNotAuthorized { .. }) + )); + assert!(matches!( + sender.make_phase_message(RbcPhase::Ready, first, 1), + Err(RbcError::PhaseNotAuthorized { .. }) + )); + + for peer in [1, 2] { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + peer, + 0, + RbcPhase::Echo, + first, + ); + sender.handle_phase(peer, message).unwrap(); + } + assert!(sender.make_phase_message(RbcPhase::Ready, first, 1).is_ok()); + } + + #[test] + fn phase_macs_work_under_every_initial_authentication_mode() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 3, 0x22); + + for authentication in [ + BlockAuthenticationScheme::Ed25519, + BlockAuthenticationScheme::MlDsa44, + BlockAuthenticationScheme::MlDsa65, + BlockAuthenticationScheme::MacVector, + ] { + let mut sender = kernel(Arc::clone(&committee), &keyrings, 0, authentication); + let mut receiver = kernel(Arc::clone(&committee), &keyrings, 1, authentication); + authorize_echo(&mut sender, block_ref); + let message = sender + .make_phase_message(RbcPhase::Echo, block_ref, 1) + .unwrap(); + assert!(receiver.handle_phase(0, message).is_ok()); + } + } + + #[test] + fn every_authenticated_phase_field_and_direct_peer_are_checked() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut sender = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + let block_ref = block(0, 5, 0x31); + authorize_echo(&mut sender, block_ref); + let valid = sender + .make_phase_message(RbcPhase::Echo, block_ref, 1) + .unwrap(); + + let mut wrong_phase = valid.clone(); + wrong_phase.phase = RbcPhase::Ready; + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 1, + BlockAuthenticationScheme::Ed25519, + ); + assert_eq!( + receiver.handle_phase(0, wrong_phase), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + + let mut wrong_digest = valid.clone(); + wrong_digest.block_ref.digest = BlockDigest::from([0x32; 32]); + assert_eq!( + receiver.handle_phase(0, wrong_digest), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + + let mut wrong_author = valid.clone(); + wrong_author.block_ref.authority = 1; + assert_eq!( + receiver.handle_phase(0, wrong_author), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + + let mut wrong_round = valid.clone(); + wrong_round.block_ref.round += 1; + assert_eq!( + receiver.handle_phase(0, wrong_round), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + + let mut wrong_sender = valid.clone(); + wrong_sender.sender = 2; + assert_eq!( + receiver.handle_phase(2, wrong_sender), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + + assert!(matches!( + receiver.handle_phase(2, valid.clone()), + Err(RbcError::SenderPeerMismatch { .. }) + )); + assert!(receiver.slots.is_empty()); + + let mut wrong_recipient = valid.clone(); + wrong_recipient.recipient = 2; + assert!(matches!( + receiver.handle_phase(0, wrong_recipient), + Err(RbcError::WrongRecipient { .. }) + )); + assert!(receiver.slots.is_empty()); + + let mut other_instance = kernel_with_instance( + Arc::clone(&committee), + &keyrings, + 1, + BlockAuthenticationScheme::Ed25519, + 0xBB, + ); + assert_eq!( + other_instance.handle_phase(0, valid.clone()), + Err(RbcError::InvalidPhaseTag) + ); + assert!(other_instance.slots.is_empty()); + + let mut other_scheme = kernel_with_instance( + Arc::clone(&committee), + &keyrings, + 1, + BlockAuthenticationScheme::MlDsa44, + TEST_INSTANCE_BYTE, + ); + assert_eq!( + other_scheme.handle_phase(0, valid.clone()), + Err(RbcError::InvalidPhaseTag) + ); + assert!(other_scheme.slots.is_empty()); + + let changed_committee = Committee::new_test(vec![2, 1, 1, 1]); + let mut other_committee = kernel( + changed_committee, + &keyrings, + 1, + BlockAuthenticationScheme::Ed25519, + ); + assert_eq!( + other_committee.handle_phase(0, valid), + Err(RbcError::InvalidPhaseTag) + ); + assert!(other_committee.slots.is_empty()); + } + + #[test] + fn initial_mac_cannot_be_substituted_for_phase_mac() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut sender = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::MacVector, + ); + let mut receiver = kernel( + committee, + &keyrings, + 1, + BlockAuthenticationScheme::MacVector, + ); + let block_ref = block(0, 5, 0x41); + authorize_echo(&mut sender, block_ref); + let mut message = sender + .make_phase_message(RbcPhase::Echo, block_ref, 1) + .unwrap(); + message.tag = sender.make_initial_mac_tag(block_ref, 1).unwrap(); + + assert_eq!( + receiver.handle_phase(0, message), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + } + + #[test] + fn truncated_phase_message_fails_deserialization() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut sender = kernel(committee, &keyrings, 0, BlockAuthenticationScheme::Ed25519); + let block_ref = block(0, 5, 0x43); + authorize_echo(&mut sender, block_ref); + let message = sender + .make_phase_message(RbcPhase::Echo, block_ref, 1) + .unwrap(); + let mut encoded = bincode::serialize(&message).unwrap(); + encoded.pop(); + + assert!(bincode::deserialize::(&encoded).is_err()); + } + + #[test] + fn symmetric_key_cannot_reflect_message_direction() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut sender = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + let block_ref = block(0, 5, 0x42); + authorize_echo(&mut sender, block_ref); + let mut reflected = sender + .make_phase_message(RbcPhase::Echo, block_ref, 1) + .unwrap(); + reflected.sender = 1; + reflected.recipient = 0; + let mut receiver = kernel(committee, &keyrings, 0, BlockAuthenticationScheme::Ed25519); + + assert_eq!( + receiver.handle_phase(1, reflected), + Err(RbcError::InvalidPhaseTag) + ); + assert!(receiver.slots.is_empty()); + } + + #[test] + fn echo_quorum_without_header_latches_fetch_then_ready() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 4, 0x51); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + + for sender in 0..2 { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Echo, + block_ref, + ); + assert!(receiver.handle_phase(sender, message).unwrap().is_empty()); + } + let message = phase_message( + Arc::clone(&committee), + &keyrings, + 2, + 3, + RbcPhase::Echo, + block_ref, + ); + let effects = receiver.handle_phase(2, message).unwrap(); + let mut expected_holders = AuthoritySet::default(); + expected_holders.insert(0); + expected_holders.insert(1); + expected_holders.insert(2); + assert_eq!( + effects, + vec![RbcEffect::NeedHeader { + block_ref, + holders: expected_holders, + }] + ); + assert_eq!( + holder_vec(receiver.header_holders(&block_ref)), + vec![0, 1, 2] + ); + + assert_eq!( + receiver.note_header_available(block_ref).unwrap(), + vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + }] + ); + } + + #[test] + fn ready_validity_without_header_can_ready_and_deliver_after_fetch() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 4, 0x52); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + + let first = phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Ready, + block_ref, + ); + assert!(receiver.handle_phase(0, first).unwrap().is_empty()); + let second = phase_message( + Arc::clone(&committee), + &keyrings, + 1, + 3, + RbcPhase::Ready, + block_ref, + ); + assert!(matches!( + receiver.handle_phase(1, second).unwrap().as_slice(), + [RbcEffect::NeedHeader { .. }] + )); + + assert_eq!( + receiver.note_header_available(block_ref).unwrap(), + vec![ + RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + }, + RbcEffect::Deliver(block_ref), + ] + ); + } + + #[test] + fn header_fetch_can_retry_and_reemits_when_holder_set_grows() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 4, 0x54); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + + let first = phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Ready, + block_ref, + ); + assert!(receiver.handle_phase(0, first).unwrap().is_empty()); + let second = phase_message( + Arc::clone(&committee), + &keyrings, + 1, + 3, + RbcPhase::Ready, + block_ref, + ); + let initial_request = receiver.handle_phase(1, second).unwrap(); + assert_eq!( + receiver.retry_header_request(block_ref).unwrap(), + initial_request.first().cloned() + ); + + let third = phase_message(committee, &keyrings, 2, 3, RbcPhase::Ready, block_ref); + let expanded_request = receiver.handle_phase(2, third).unwrap(); + assert!(matches!( + expanded_request.as_slice(), + [RbcEffect::NeedHeader { holders, .. }] if holder_vec(*holders) == vec![0, 1, 2] + )); + assert_eq!( + receiver.retry_header_request(block_ref).unwrap(), + expanded_request.first().cloned() + ); + + receiver.note_header_available(block_ref).unwrap(); + assert_eq!(receiver.retry_header_request(block_ref).unwrap(), None); + } + + #[test] + fn ready_quorum_never_delivers_before_header_arrives() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 4, 0x53); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + + for sender in 0..3 { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Ready, + block_ref, + ); + receiver.handle_phase(sender, message).unwrap(); + } + assert_eq!(receiver.slot(&block_ref).unwrap().delivered, None); + + assert_eq!( + receiver.note_header_available(block_ref).unwrap(), + vec![ + RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + }, + RbcEffect::Deliver(block_ref), + ] + ); + } + + #[test] + fn duplicates_do_not_inflate_phase_stake() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 6, 0x61); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + receiver.note_header_available(block_ref).unwrap(); + + for sender in [0, 0, 1] { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Echo, + block_ref, + ); + assert!(receiver.handle_phase(sender, message).unwrap().is_empty()); + } + let message = phase_message( + Arc::clone(&committee), + &keyrings, + 2, + 3, + RbcPhase::Echo, + block_ref, + ); + assert!(matches!( + receiver.handle_phase(2, message).unwrap().as_slice(), + [RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + .. + }] + )); + } + + #[test] + fn slot_global_echo_does_not_first_seen_lock_other_candidate() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let first = block(0, 8, 0x71); + let quorum_candidate = block(0, 8, 0x72); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + receiver.note_header_available(first).unwrap(); + assert!(matches!( + receiver.authorize_echo(first).unwrap().as_slice(), + [RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + .. + }] + )); + receiver.note_header_available(quorum_candidate).unwrap(); + assert!( + receiver + .authorize_echo(quorum_candidate) + .unwrap() + .is_empty() + ); + + let mut last_effects = Vec::new(); + for sender in 1..4 { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 0, + RbcPhase::Echo, + quorum_candidate, + ); + last_effects = receiver.handle_phase(sender, message).unwrap(); + } + assert_eq!( + last_effects, + vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref: quorum_candidate, + }] + ); + let slot = receiver.slot(&first).unwrap(); + assert_eq!(slot.echoed, Some(first)); + assert_eq!(slot.readied, Some(quorum_candidate)); + } + + #[test] + fn evidence_is_per_candidate_but_delivery_is_slot_global() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let first = block(0, 10, 0x81); + let second = block(0, 10, 0x82); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + receiver.note_header_available(first).unwrap(); + receiver.note_header_available(second).unwrap(); + + for candidate in [first, second] { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Echo, + candidate, + ); + receiver.handle_phase(0, message).unwrap(); + } + assert!(receiver.candidate(&first).unwrap().echoes.votes.contains(0)); + assert!( + receiver + .candidate(&second) + .unwrap() + .echoes + .votes + .contains(0) + ); + + for sender in 0..2 { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Ready, + first, + ); + receiver.handle_phase(sender, message).unwrap(); + } + assert_eq!(receiver.slot(&first).unwrap().delivered, Some(first)); + + for sender in 0..3 { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Ready, + second, + ); + assert!(receiver.handle_phase(sender, message).unwrap().is_empty()); + } + assert_eq!(receiver.slot(&second).unwrap().delivered, Some(first)); + } + + #[test] + fn four_kernel_split_initial_values_converge_on_at_most_one_delivery() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let first = block(0, 12, 0xA1); + let conflicting = block(0, 12, 0xA2); + let mut kernels: Vec<_> = (0..4) + .map(|authority| { + kernel( + Arc::clone(&committee), + &keyrings, + authority, + BlockAuthenticationScheme::Ed25519, + ) + }) + .collect(); + + let mut initial_effects = Vec::new(); + for (authority, block_ref) in [(0, first), (1, first), (2, first), (3, conflicting)] { + kernels[authority as usize] + .note_header_available(block_ref) + .unwrap(); + let effects = kernels[authority as usize] + .authorize_echo(block_ref) + .unwrap(); + initial_effects.push((authority, effects)); + } + + let deliveries = pump_phase_effects(&mut kernels, initial_effects, true); + assert!(deliveries.iter().all(|delivered| delivered == &[first])); + assert!( + deliveries + .iter() + .flatten() + .all(|delivered| *delivered != conflicting) + ); + assert_eq!(kernels[3].slot(&first).unwrap().echoed, Some(conflicting)); + assert_eq!(kernels[3].slot(&first).unwrap().readied, Some(first)); + } + + #[test] + fn poisoned_initial_mac_does_not_block_rbc_totality() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(0, 13, 0xA3); + let mut kernels: Vec<_> = (0..4) + .map(|authority| { + kernel( + Arc::clone(&committee), + &keyrings, + authority, + BlockAuthenticationScheme::MacVector, + ) + }) + .collect(); + + let valid_for_one = kernels[0].make_initial_mac_tag(block_ref, 1).unwrap(); + let valid_for_two = kernels[0].make_initial_mac_tag(block_ref, 2).unwrap(); + kernels[1] + .verify_initial_mac_tag(0, block_ref, &valid_for_one) + .unwrap(); + kernels[2] + .verify_initial_mac_tag(0, block_ref, &valid_for_two) + .unwrap(); + assert_eq!( + kernels[3].verify_initial_mac_tag(0, block_ref, &valid_for_two), + Err(RbcError::InvalidInitialTag) + ); + + let mut initial_effects = Vec::new(); + for authority in 0..3 { + kernels[authority as usize] + .note_header_available(block_ref) + .unwrap(); + let effects = kernels[authority as usize] + .authorize_echo(block_ref) + .unwrap(); + initial_effects.push((authority, effects)); + } + + let deliveries = pump_phase_effects(&mut kernels, initial_effects, true); + assert!(deliveries.iter().all(|delivered| delivered == &[block_ref])); + let recovered_slot = kernels[3].slot(&block_ref).unwrap(); + assert_eq!(recovered_slot.echoed, None); + assert_eq!(recovered_slot.readied, Some(block_ref)); + assert_eq!(recovered_slot.delivered, Some(block_ref)); + } + + #[test] + fn weighted_thresholds_and_author_echo_are_counted_exactly() { + let committee = Committee::new_test(vec![3, 2, 1, 1]); + assert_eq!(committee.validity_threshold(), 3); + assert_eq!(committee.quorum_threshold(), 5); + let keyrings = mac_keyrings_for_test(4); + let block_ref = block(3, 11, 0x91); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + receiver.note_header_available(block_ref).unwrap(); + assert_eq!( + receiver.authorize_echo(block_ref).unwrap(), + vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, + }] + ); + + let high_stake = phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Echo, + block_ref, + ); + assert!(receiver.handle_phase(0, high_stake).unwrap().is_empty()); + let boundary = phase_message(committee, &keyrings, 2, 3, RbcPhase::Echo, block_ref); + assert!(matches!( + receiver.handle_phase(2, boundary).unwrap().as_slice(), + [RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + .. + }] + )); + } +} diff --git a/docs/starfish-rbc-protocol.md b/docs/starfish-rbc-protocol.md index 26cf7001..216102b7 100644 --- a/docs/starfish-rbc-protocol.md +++ b/docs/starfish-rbc-protocol.md @@ -1,6 +1,6 @@ # Starfish-RBC protocol specification -Status: design milestone, not implemented +Status: protocol specification with isolated RBC kernel; network and DAG integration pending This document specifies the first correctness-oriented prototype of Starfish with reliable header certification and a signature-free MAC configuration. The provisional CLI name is `starfish-rbc`. @@ -10,10 +10,10 @@ availability, DAG ordering, and commitment, while adding a Bracha reliable-broad block headers. It uses the same validator, committee, networking, storage, orchestrator, and benchmark setup as the other protocols in this repository. -This document is not a claim that the protocol is already proved or implemented. The motivating -work-in-progress note gives reliable-delivery and MAC-vector ingredients, but does not prove their -composition with Starfish. The proof obligations below must be discharged before making a safety -or liveness claim. +This document is not a claim that the composed protocol is already proved or fully implemented. The +motivating work-in-progress note gives reliable-delivery and MAC-vector ingredients, but does not +prove their composition with Starfish. The proof obligations below must be discharged before making +a safety or liveness claim. ## 1. Goals @@ -96,19 +96,39 @@ One reliable-broadcast instance exists for each slot: Slot = (protocol_instance, committee_id, author, round) ``` -The value proposed in a slot is a canonical Starfish header. Its identifier is the existing: +The value proposed in a slot is a canonical Starfish header. Its identifier remains: ```text BlockReference = (author, round, content_digest) ``` -`content_digest` commits to the existing canonical header content, including the transaction -commitment, parent references, acknowledgments, and other consensus-relevant fields. It does not -commit to the initial authentication sidecar or to reliable-broadcast messages. - -`protocol_instance` and `committee_id` are domain-separation inputs. They are not added to -`BlockReference`. Their exact derivation must be fixed before implementation; a committee/genesis -identifier is preferable to a human-readable CLI string. +`content_digest` must commit unambiguously to the canonical header content, including the +transaction commitment, parent references, acknowledgments, and other consensus-relevant fields. +It does not commit to the initial authentication sidecar or to reliable-broadcast messages. + +The current shared `BlockDigest` implementation hashes the parent-reference vector immediately +followed by the acknowledgment vector without encoding their lengths. Moving a reference across +that boundary can therefore preserve the digest without finding a BLAKE3 collision. Before +Starfish-RBC accepts headers, milestone three must add a Starfish-RBC canonical content encoding +with explicit field markers and collection lengths. This is a blocking proof obligation, not an +optional optimization. + +`protocol_instance` and `committee_id` are authenticated domain-separation inputs but are not added +to `BlockReference`: + +- `protocol_instance` is a nonzero 32-byte execution/session identifier shared through genesis + configuration. A fixed version string is not sufficient because benchmark keys are deterministic + and reused across independent runs. +- `committee_id` is a 32-byte BLAKE3 derive-key hash under context + `STARFISH_RBC_V1_COMMITTEE_ID`. Its canonical input contains the committee length, effective + validity and quorum thresholds, Reed-Solomon information length, the three stored optimistic + thresholds, and each authority in index order with its index, stake, and Ed25519, BLS, + ML-DSA-44, and ML-DSA-65 public keys. It excludes addresses, timeouts, private keys, and pairwise + MAC keys. + +The kernel rejects an all-zero protocol instance both at construction and deserialization, a +keyring-length mismatch, committees above `MAX_COMMITTEE_SIZE`, and deserialized committees whose +effective validity/quorum thresholds or information length disagree with the canonical formulas. Header processing has three distinct gates: @@ -125,14 +145,25 @@ The implementation must not keep these gates bundled in the current all-or-nothi ### 3.2 Initial header authentication -The configured block-authentication method applies only to the author's initial header proof: +The configured block-authentication method applies only to the author's initial header proof. Its +stable authentication code is Ed25519 `0x00`, ML-DSA-44 `0x01`, ML-DSA-65 `0x02`, or MAC `0x03`. -- Ed25519 and ML-DSA sign a domain-separated statement containing the slot and content digest. -- In MAC mode, author `A` creates one receiver-specific tag for each validator `Q`: +Ed25519 and ML-DSA sign the BLAKE3 digest of the 119-byte base statement below. In MAC mode, author +`A` creates one receiver-specific tag for each validator `Q` over the 123-byte MAC statement: ```text -InitialTag(A, Q, block_ref) = - MAC[k(A,Q)](INITIAL || Slot || block_ref || A || Q) +base_statement = + "STARFISH_RBC_V1" // 15 bytes + || kind // INITIAL = 0x00 + || initial_authentication // 1 byte + || protocol_instance // 32 bytes + || committee_id // 32 bytes + || author // u16, big-endian + || round // u32, big-endian + || content_digest // 32 bytes + +InitialSignatureDigest = BLAKE3(base_statement) +InitialTag(A, Q) = MAC[k(A,Q)](base_statement || A:u16_be || Q:u16_be) ``` The ordered collection of these tags is the conceptual MAC vector. With direct dissemination, @@ -148,15 +179,21 @@ the header clean, globally available, or safe to commit. ### 3.3 Phase-message authentication ECHO and READY use pairwise MAC authentication for every initial header-authentication variant. -For phase sender `S` and recipient `Q`: +Their kind codes are ECHO `0x01` and READY `0x02`. For phase sender `S` and recipient `Q`: ```text PhaseTag(S, Q, phase, block_ref) = - MAC[k(S,Q)](RBC_PHASE || Slot || phase || block_ref || S || Q) + MAC[k(S,Q)]( + base_statement(phase, block_ref) + || S:u16_be + || Q:u16_be + ) ``` -The concrete encoding must use fixed-width, canonical fields and distinct domain values for -INITIAL, ECHO, and READY. The protocol must not rely on ambiguous string concatenation. +The resulting phase statement is exactly 123 bytes. The selected initial-authentication mode is +bound even though every mode uses phase MACs; nodes with inconsistent modes cannot combine +transcripts. Enum discriminants, bincode, YAML, and ambiguous string concatenation are never used +as authenticated bytes. An inbound phase message counts only if all of the following hold: @@ -171,6 +208,11 @@ A forwarded or replayed phase message received from another peer never counts, e contain a valid tag addressed to the receiver. Local ECHO/READY actions are counted locally and do not require a loopback network message. +The isolated kernel enforces committee membership and rejects genesis slots. The active/retained +round-window check requires the service's current round and is therefore an explicit ingress +adapter responsibility in milestone three; it must run before calling the kernel or allocating a +candidate. + ## 4. Messages Version one uses the following logical messages: @@ -182,19 +224,11 @@ HeaderProposal { initial_authentication, } -Echo { - slot, - block_ref, - sender, - recipient, - phase_tag, -} - -Ready { - slot, +RbcPhaseMessage { block_ref, sender, recipient, + phase, phase_tag, } @@ -209,9 +243,14 @@ HeaderResponse { } ``` -ECHO and READY contain a block reference, not the header. This avoids rebroadcasting each header -quadratically. Header request/response traffic transports data only and is never counted as quorum -testimony. +The protocol instance, committee ID, and authentication mode are fixed service context and need not +be repeated on the wire, but every tag authenticates them. ECHO and READY contain a block reference, +not the header. This avoids rebroadcasting each header quadratically. Header request/response +traffic transports data only and is never counted as quorum testimony. + +The milestone-two phase message has a golden bincode regression vector. The eventual +`NetworkMessage` integration must append a new variant or use a versioned envelope; it must not +silently change existing variant discriminants. An honest ECHO or READY sender must possess the matching content-validated header. Consequently, when a threshold is observed before the local header arrives, the receiver can request the header @@ -251,6 +290,13 @@ CandidateState { } ``` +The isolated milestone-two kernel temporarily represents header presence with an internal boolean +to test transitions. Its header-available and initial-proof authorization hooks are module-private, +so another crate module cannot assert these facts by call ordering. Milestone three must replace the +boolean with a typed handle that owns or pins the canonical content-validated header, and expose +only combined typed ingress paths. Cache eviction must not leave the kernel advertising a header it +no longer retains. + The `echoed`, `readied`, and `delivered` guards are slot-global, not per digest. Evidence is kept per digest so that Byzantine equivocation can be observed without locking the receiver to the first value it sees. @@ -271,7 +317,10 @@ bounds are deferred with crash/restart support; arbitrary cache eviction is not Sending a local phase is one atomic state transition: set the slot-global guard, insert the local authority into that candidate's phase-sender set, and enqueue separately authenticated messages for -all other validators. Threshold checks include this local evidence. +all other validators. Threshold checks include this local evidence. Message materialization checks +the recorded slot-global guard and the kernel's header-present predicate again; it cannot +authenticate a phase for an unauthorized or conflicting value. Milestone three makes that predicate +a pinned-header handle. The same authorized phase may be materialized again for retransmission. ## 6. State machine @@ -345,6 +394,12 @@ Every honest ECHO and READY sender is a header holder. Retrieval therefore proce 4. Re-evaluate all latched ECHO/READY triggers immediately after storing the header. 5. Continue requesting while threshold progress is blocked and untried holders remain. +`NeedHeader` is a recovery wake-up, not a one-shot delivery assumption. The kernel re-emits it when +the authenticated holder set grows and exposes the current effect to a durable retry timer. The +integration layer must keep that timer active, retry or fan out after loss or an unresponsive +holder, and cancel it only after a matching content-validated header is pinned or the instance is +safely retired. + At least one honest holder exists in every `V`-stake READY set. At least `f + 1` honest holders exist in an equal-stake `2f + 1` ECHO quorum. Byzantine responses can delay retrieval but cannot change the accepted content. An honest validator that sends ECHO or READY retains the canonical header @@ -517,6 +572,9 @@ tests. ### 11.1 RBC unit tests +The milestone-two suite includes deterministic four-kernel message-pump traces for split initial +values and poisoned-recipient MAC recovery, in addition to the local transition tests below. + - `n = 4, f = 1`: a Byzantine author gives candidate X to one honest validator and candidate Y to two others. A validator that first saw X must still process quorum traffic for Y. - A Byzantine author provides valid initial tags to enough validators to form an ECHO quorum but @@ -616,12 +674,14 @@ RBC copy of every protocol. Each milestone is committed separately. -1. **Specification:** this document, with no protocol code changes. -2. **RBC kernel:** domain-separated phase MACs, slot-global ECHO/READY state, per-value evidence, - header-holder tracking, and adversarial unit tests. +1. **Specification (complete):** this document, initially with no protocol code changes. +2. **RBC kernel (complete):** domain-separated initial/phase statements, explicit session and + committee identity, slot-global ECHO/READY state, guarded recipient-specific message + materialization, retryable header-holder tracking, and local plus four-kernel adversarial tests. + The synchronous kernel is intentionally not network- or DAG-wired yet. 3. **Header staging and retrieval:** split content validation from initial authentication, add - fixed-size-checked candidate staging with pending triggers, and fetch headers from direct phase - senders. + a length-delimited Starfish-RBC content digest, typed/pinned fixed-size-checked candidate staging, + retained-window filtering, pending triggers, and durable fetching from direct phase senders. 4. **Certified Starfish integration:** add `starfish-rbc`, selectable initial authentication, dirty/clean lifecycle, clean-only acknowledgments, and clean-only consensus/linearization. 5. **End-to-end validation:** poisoned-tag, equivocation, dangling-parent, and all-authentication @@ -632,16 +692,18 @@ Each milestone is committed separately. and restart tests. 8. **Benchmarks:** direct and tree comparisons with results reported outside this specification. -## 15. Decisions still required before implementation +## 15. Remaining integration decisions -The protocol behavior above is fixed, but implementation must still choose: +The kernel behavior and authenticated encoding above are fixed. Integration must still choose: -- the exact `protocol_instance` and `committee_id` derivation; -- the canonical byte encoding for phase-MAC statements; +- how benchmark genesis generates and distributes the fresh 32-byte `protocol_instance`; +- the exact field markers and collection-length widths for the Starfish-RBC content digest; - a safe post-v1 state-retirement and garbage-collection rule; - header-holder request fanout and retry timing; -- the final CLI spelling for MAC authentication; and - whether legacy unsafe `*-mac` aliases are renamed or retained as lower-bound benchmarks. +The authentication selector remains `--block-authentication`; Starfish-RBC integration adds `mac` +to the existing Ed25519, ML-DSA-44, and ML-DSA-65 values while retaining Ed25519 as the default. + None of these choices may weaken the slot-global phase guards, direct-message checks, header availability requirement, or clean-only Starfish boundary. From 1a20b540bf77ef4c7e17619fde922a18e172e352 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:08:21 +0200 Subject: [PATCH 23/62] Add canonical Starfish-RBC header boundary --- crates/starfish-core/src/crypto.rs | 52 + crates/starfish-core/src/starfish_rbc.rs | 2567 ++++++++++++++++++++-- crates/starfish-core/src/types.rs | 4 +- docs/starfish-rbc-protocol.md | 184 +- 4 files changed, 2616 insertions(+), 191 deletions(-) diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index df659c4f..8fe6bbf6 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -220,6 +220,58 @@ impl BlockDigest { ) } + /// Canonical Starfish-RBC header-content digest. + /// + /// This encoding is intentionally separate from the legacy block digest: + /// every field is tagged, collection lengths are explicit, integers are + /// big-endian, and the transaction commitment is mandatory. Authentication + /// and protocol sidecars are not part of the content identity. + pub(crate) fn new_starfish_rbc_header( + authority: AuthorityIndex, + round: RoundNumber, + block_references: &[BlockReference], + acknowledgment_references: &[BlockReference], + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + ) -> Self { + const AUTHORITY_FIELD: u8 = 0x01; + const ROUND_FIELD: u8 = 0x02; + const PARENTS_FIELD: u8 = 0x03; + const ACKNOWLEDGMENTS_FIELD: u8 = 0x04; + const CREATION_TIME_FIELD: u8 = 0x05; + const TRANSACTIONS_COMMITMENT_FIELD: u8 = 0x06; + + fn hash_reference(hasher: &mut Blake3Hasher, block_ref: &BlockReference) { + hasher.update(&block_ref.authority.to_be_bytes()); + hasher.update(&block_ref.round.to_be_bytes()); + hasher.update(block_ref.digest.as_ref()); + } + + fn hash_references(hasher: &mut Blake3Hasher, references: &[BlockReference]) { + let length = + u32::try_from(references.len()).expect("Starfish-RBC reference count exceeds u32"); + hasher.update(&length.to_be_bytes()); + for block_ref in references { + hash_reference(hasher, block_ref); + } + } + + let mut hasher = Blake3Hasher::new(); + hasher.update(&[AUTHORITY_FIELD]); + hasher.update(&authority.to_be_bytes()); + hasher.update(&[ROUND_FIELD]); + hasher.update(&round.to_be_bytes()); + hasher.update(&[PARENTS_FIELD]); + hash_references(&mut hasher, block_references); + hasher.update(&[ACKNOWLEDGMENTS_FIELD]); + hash_references(&mut hasher, acknowledgment_references); + hasher.update(&[CREATION_TIME_FIELD]); + hasher.update(&meta_creation_time_ns.to_be_bytes()); + hasher.update(&[TRANSACTIONS_COMMITMENT_FIELD]); + hasher.update(transactions_commitment.as_ref()); + Self(hasher.finalize().into()) + } + pub fn new_without_transactions_with_unprovable( authority: AuthorityIndex, round: RoundNumber, diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index d49ee2a3..261bdf45 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -11,15 +11,19 @@ use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; -use ahash::AHashMap; -use serde::{Deserialize, Deserializer, Serialize, de}; +use ahash::{AHashMap, AHashSet}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use crate::{ committee::{Committee, QuorumThreshold, StakeAggregator, ValidityThreshold}, - crypto::{Blake3Hasher, MacKey, MacTag}, + crypto::{ + Blake3Hasher, MacKey, MacTag, MlDsa44SignatureBytes, MlDsa65SignatureBytes, SignatureBytes, + TransactionsCommitment, + }, types::{ - AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockReference, - MAX_COMMITTEE_SIZE, RoundNumber, Stake, + AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, + BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, + TimestampNs, compress_acknowledgments, expand_acknowledgments, }, }; @@ -39,6 +43,476 @@ const BASE_STATEMENT_SIZE: usize = PROTOCOL_DOMAIN.len() + COMMITTEE_ID_SIZE + BLOCK_REFERENCE_SIZE; const MAC_STATEMENT_SIZE: usize = BASE_STATEMENT_SIZE + 2 + 2; +const RBC_BLOCK_REFERENCE_SIZE: usize = 2 + 4 + 32; +const RBC_HEADER_FIXED_CONTENT_SIZE: usize = 1 + 2 + 1 + 4 + 1 + 4 + 1 + 4 + 1 + 8 + 1 + 32; +const MAX_RBC_HEADER_CONTENT_SIZE: usize = 4 * 1024 * 1024; +const MAX_RBC_REFERENCES_PER_FIELD: usize = u16::MAX as usize; +const MAX_RBC_FUTURE_ROUNDS: RoundNumber = 100; + +mod bounded_references { + use std::{fmt, marker::PhantomData}; + + use serde::de::{Error as _, SeqAccess, Visitor}; + + use super::*; + + pub(super) fn serialize( + references: &[BlockReference], + serializer: S, + ) -> Result + where + S: Serializer, + { + references.serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + struct ReferencesVisitor(PhantomData); + + impl<'de> Visitor<'de> for ReferencesVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "at most {MAX_RBC_REFERENCES_PER_FIELD} Starfish-RBC references" + ) + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let size_hint = sequence.size_hint().unwrap_or(0); + if size_hint > MAX_RBC_REFERENCES_PER_FIELD { + return Err(A::Error::custom(format!( + "RBC reference count {size_hint} exceeds {MAX_RBC_REFERENCES_PER_FIELD}" + ))); + } + let mut references = Vec::with_capacity(size_hint); + while let Some(reference) = sequence.next_element()? { + if references.len() == MAX_RBC_REFERENCES_PER_FIELD { + return Err(A::Error::custom(format!( + "RBC reference count exceeds {MAX_RBC_REFERENCES_PER_FIELD}" + ))); + } + references.push(reference); + } + Ok(references) + } + } + + deserializer.deserialize_seq(ReferencesVisitor(PhantomData)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct RbcAckFields { + intersection: Option, + #[serde(with = "bounded_references")] + extra_references: Vec, +} + +impl RbcAckFields { + fn from_logical( + block_references: &[BlockReference], + acknowledgment_references: &[BlockReference], + ) -> Self { + let (intersection, extra_references) = + compress_acknowledgments(block_references, acknowledgment_references); + Self { + intersection, + extra_references, + } + } + + fn logical(&self, block_references: &[BlockReference]) -> Vec { + expand_acknowledgments(block_references, self.intersection, &self.extra_references) + } + + fn is_canonical(&self, block_references: &[BlockReference]) -> bool { + if self + .intersection + .is_some_and(|start| start as usize > block_references.len()) + { + return false; + } + let logical = self.logical(block_references); + let (intersection, extra_references) = compress_acknowledgments(block_references, &logical); + self.intersection == intersection && self.extra_references == extra_references + } +} + +/// Authentication-free, canonical Starfish-RBC header content. +/// +/// Acknowledgments stay canonically compressed on wire, but the digest hashes +/// their expanded logical vector with an explicit boundary from parents. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct RbcCanonicalHeader { + reference: BlockReference, + #[serde(with = "bounded_references")] + block_references: Vec, + acknowledgments: RbcAckFields, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, +} + +impl RbcCanonicalHeader { + fn try_new( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + ) -> Result { + for (field, count) in [ + ("parent", block_references.len()), + ("acknowledgment", acknowledgment_references.len()), + ] { + if count > MAX_RBC_REFERENCES_PER_FIELD { + return Err(RbcError::TooManyHeaderReferences { + field, + count, + maximum: MAX_RBC_REFERENCES_PER_FIELD, + }); + } + } + let mut parent_set = AHashSet::new(); + for parent in &block_references { + if !parent_set.insert(*parent) { + return Err(RbcError::DuplicateParent(*parent)); + } + } + let mut acknowledgment_set = AHashSet::new(); + for acknowledgment in &acknowledgment_references { + if !acknowledgment_set.insert(*acknowledgment) { + return Err(RbcError::DuplicateAcknowledgment(*acknowledgment)); + } + } + + let acknowledgments = + RbcAckFields::from_logical(&block_references, &acknowledgment_references); + let logical_acknowledgments = acknowledgments.logical(&block_references); + let reference = BlockReference { + authority, + round, + digest: BlockDigest::new_starfish_rbc_header( + authority, + round, + &block_references, + &logical_acknowledgments, + meta_creation_time_ns, + transactions_commitment, + ), + }; + let header = Self { + reference, + block_references, + acknowledgments, + meta_creation_time_ns, + transactions_commitment, + }; + if header.encoded_content_size(logical_acknowledgments.len())? > MAX_RBC_HEADER_CONTENT_SIZE + { + return Err(RbcError::HeaderContentTooLarge); + } + Ok(header) + } + + pub(crate) fn from_block_header(header: &BlockHeader) -> Result { + if header.strong_vote.is_some() + || header.bls.is_some() + || header.sailfish.is_some() + || header.unprovable_certificate.is_some() + { + return Err(RbcError::ForbiddenHeaderExtensions); + } + let Some(acknowledgments) = header.ack.as_ref() else { + return Err(RbcError::MissingAcknowledgments); + }; + let Some(transactions_commitment) = header.transactions_commitment else { + return Err(RbcError::MissingTransactionsCommitment); + }; + let parent_count = header.block_references.len(); + let extra_acknowledgment_count = acknowledgments.extra_references.len(); + for (field, count) in [ + ("parent", parent_count), + ("acknowledgment", extra_acknowledgment_count), + ] { + if count > MAX_RBC_REFERENCES_PER_FIELD { + return Err(RbcError::TooManyHeaderReferences { + field, + count, + maximum: MAX_RBC_REFERENCES_PER_FIELD, + }); + } + } + let intersection_start = match acknowledgments.intersection { + Some(start) if start as usize <= parent_count => start as usize, + Some(_) => return Err(RbcError::NonCanonicalAcknowledgments), + None => parent_count, + }; + let logical_acknowledgment_count = parent_count + .checked_sub(intersection_start) + .and_then(|count| count.checked_add(extra_acknowledgment_count)) + .ok_or(RbcError::HeaderContentTooLarge)?; + if logical_acknowledgment_count > MAX_RBC_REFERENCES_PER_FIELD { + return Err(RbcError::TooManyHeaderReferences { + field: "acknowledgment", + count: logical_acknowledgment_count, + maximum: MAX_RBC_REFERENCES_PER_FIELD, + }); + } + let reference_count = parent_count + .checked_add(logical_acknowledgment_count) + .ok_or(RbcError::HeaderContentTooLarge)?; + let encoded_size = RBC_BLOCK_REFERENCE_SIZE + .checked_mul(reference_count) + .and_then(|size| size.checked_add(RBC_HEADER_FIXED_CONTENT_SIZE)) + .ok_or(RbcError::HeaderContentTooLarge)?; + if encoded_size > MAX_RBC_HEADER_CONTENT_SIZE { + return Err(RbcError::HeaderContentTooLarge); + } + Ok(Self { + reference: header.reference, + block_references: header.block_references.clone(), + acknowledgments: RbcAckFields { + intersection: acknowledgments.intersection, + extra_references: acknowledgments.extra_references.clone(), + }, + meta_creation_time_ns: header.meta_creation_time_ns, + transactions_commitment, + }) + } + + pub(crate) fn reference(&self) -> BlockReference { + self.reference + } + + pub(crate) fn block_references(&self) -> &[BlockReference] { + &self.block_references + } + + pub(crate) fn acknowledgment_references(&self) -> Vec { + self.acknowledgments.logical(&self.block_references) + } + + pub(crate) fn acknowledgment_fields(&self) -> AckFields { + AckFields { + intersection: self.acknowledgments.intersection, + extra_references: self.acknowledgments.extra_references.clone(), + } + } + + pub(crate) fn meta_creation_time_ns(&self) -> TimestampNs { + self.meta_creation_time_ns + } + + pub(crate) fn transactions_commitment(&self) -> TransactionsCommitment { + self.transactions_commitment + } + + fn encoded_content_size(&self, acknowledgment_count: usize) -> Result { + let reference_count = self + .block_references + .len() + .checked_add(acknowledgment_count) + .ok_or(RbcError::HeaderContentTooLarge)?; + RBC_BLOCK_REFERENCE_SIZE + .checked_mul(reference_count) + .and_then(|size| size.checked_add(RBC_HEADER_FIXED_CONTENT_SIZE)) + .ok_or(RbcError::HeaderContentTooLarge) + } +} + +/// An intrinsically validated header retained by `Arc` for as long as the RBC +/// state may advertise this validator as a holder. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedRbcHeader { + header: Arc, + committee_id: RbcCommitteeId, +} + +impl PinnedRbcHeader { + fn validate_with_committee_id( + header: RbcCanonicalHeader, + committee: &Committee, + committee_id: RbcCommitteeId, + ) -> Result { + let block_ref = header.reference; + if block_ref.round == 0 { + return Err(RbcError::GenesisSlot); + } + if !committee.known_authority(block_ref.authority) { + return Err(RbcError::UnknownAuthority(block_ref.authority)); + } + if !header + .acknowledgments + .is_canonical(&header.block_references) + { + return Err(RbcError::NonCanonicalAcknowledgments); + } + + let acknowledgments = header.acknowledgment_references(); + for (field, count) in [ + ("parent", header.block_references.len()), + ("acknowledgment", acknowledgments.len()), + ] { + if count > MAX_RBC_REFERENCES_PER_FIELD { + return Err(RbcError::TooManyHeaderReferences { + field, + count, + maximum: MAX_RBC_REFERENCES_PER_FIELD, + }); + } + } + if header.encoded_content_size(acknowledgments.len())? > MAX_RBC_HEADER_CONTENT_SIZE { + return Err(RbcError::HeaderContentTooLarge); + } + + let mut parent_set = AHashSet::new(); + let mut previous_round_parents = StakeAggregator::::new(); + for parent in &header.block_references { + if !committee.known_authority(parent.authority) { + return Err(RbcError::UnknownAuthority(parent.authority)); + } + if parent.round >= block_ref.round { + return Err(RbcError::ParentNotPast(*parent)); + } + if !parent_set.insert(*parent) { + return Err(RbcError::DuplicateParent(*parent)); + } + if parent.round + 1 == block_ref.round { + previous_round_parents.add(parent.authority, committee); + } + } + if !previous_round_parents.is_quorum(committee) { + return Err(RbcError::InvalidThresholdClock); + } + + let mut acknowledgment_set = AHashSet::new(); + for acknowledgment in &acknowledgments { + if !committee.known_authority(acknowledgment.authority) { + return Err(RbcError::UnknownAuthority(acknowledgment.authority)); + } + if acknowledgment.round > block_ref.round { + return Err(RbcError::AcknowledgmentFromFuture(*acknowledgment)); + } + if !acknowledgment_set.insert(*acknowledgment) { + return Err(RbcError::DuplicateAcknowledgment(*acknowledgment)); + } + } + + let expected_digest = BlockDigest::new_starfish_rbc_header( + block_ref.authority, + block_ref.round, + &header.block_references, + &acknowledgments, + header.meta_creation_time_ns, + header.transactions_commitment, + ); + if expected_digest != block_ref.digest { + return Err(RbcError::HeaderDigestMismatch { + expected: expected_digest, + actual: block_ref.digest, + }); + } + Ok(Self { + header: Arc::new(header), + committee_id, + }) + } + + #[cfg(test)] + fn validate(header: RbcCanonicalHeader, committee: &Committee) -> Result { + validate_committee(committee)?; + let committee_id = RbcCommitteeId::derive(committee)?; + Self::validate_with_committee_id(header, committee, committee_id) + } + + pub(crate) fn reference(&self) -> BlockReference { + self.header.reference + } + + pub(crate) fn header(&self) -> &RbcCanonicalHeader { + &self.header + } + + fn ensure_committee(&self, expected: RbcCommitteeId) -> Result<(), RbcError> { + if self.committee_id != expected { + return Err(RbcError::PinnedHeaderCommitteeMismatch); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum RbcInitialProof { + Ed25519(SignatureBytes), + MlDsa44(MlDsa44SignatureBytes), + MlDsa65(MlDsa65SignatureBytes), + Mac(MacTag), +} + +impl RbcInitialProof { + pub(crate) fn from_block_authentication( + authentication: &BlockAuthentication, + ) -> Result { + match authentication { + BlockAuthentication::Ed25519(signature) => Ok(Self::Ed25519(*signature)), + BlockAuthentication::MlDsa44(signature) => Ok(Self::MlDsa44(signature.clone())), + BlockAuthentication::MlDsa65(signature) => Ok(Self::MlDsa65(signature.clone())), + BlockAuthentication::MacTag(tag) => Ok(Self::Mac(*tag)), + BlockAuthentication::None | BlockAuthentication::MacVector(_) => { + Err(RbcError::InvalidInitialProof) + } + } + } +} + +/// Capability proving that the pinned header had a valid local-construction +/// path or a direct-author initial proof. Only this type can authorize ECHO. +#[derive(Debug, Eq, PartialEq)] +pub(crate) struct EchoEligibleHeader { + header: PinnedRbcHeader, + context: RbcContext, + recipient: AuthorityIndex, +} + +#[derive(Debug)] +#[must_use] +pub(crate) struct RbcLocalInitial { + header: PinnedRbcHeader, + effects: Vec, + context: RbcContext, + author: AuthorityIndex, +} + +impl RbcLocalInitial { + pub(crate) fn header(&self) -> &RbcCanonicalHeader { + self.header.header() + } + + pub(crate) fn into_parts(self) -> (PinnedRbcHeader, Vec) { + (self.header, self.effects) + } +} + +#[derive(Debug)] +#[must_use] +pub(crate) enum RbcInitialHeaderOutcome { + Authenticated { + effects: Vec, + }, + StagedUnauthenticated { + effects: Vec, + error: RbcError, + }, +} #[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize)] pub(crate) struct RbcProtocolInstanceId([u8; PROTOCOL_INSTANCE_SIZE]); @@ -201,7 +675,7 @@ pub(crate) enum RbcEffect { block_ref: BlockReference, holders: AuthoritySet, }, - Deliver(BlockReference), + Deliver(PinnedRbcHeader), } #[derive(Clone, Debug, Eq, PartialEq)] @@ -230,6 +704,55 @@ pub(crate) enum RbcError { }, UnknownAuthority(AuthorityIndex), GenesisSlot, + FutureRound { + round: RoundNumber, + maximum: RoundNumber, + }, + StaleRound { + round: RoundNumber, + minimum: RoundNumber, + }, + RoundRegression { + current: RoundNumber, + proposed: RoundNumber, + }, + RetainedRoundRegression { + current: RoundNumber, + proposed: RoundNumber, + }, + RetainedRoundAheadOfLocal { + local: RoundNumber, + proposed: RoundNumber, + }, + MissingAcknowledgments, + NonCanonicalAcknowledgments, + MissingTransactionsCommitment, + ForbiddenHeaderExtensions, + TooManyHeaderReferences { + field: &'static str, + count: usize, + maximum: usize, + }, + HeaderContentTooLarge, + ParentNotPast(BlockReference), + DuplicateParent(BlockReference), + AcknowledgmentFromFuture(BlockReference), + DuplicateAcknowledgment(BlockReference), + InvalidThresholdClock, + HeaderDigestMismatch { + expected: BlockDigest, + actual: BlockDigest, + }, + PinnedHeaderCommitteeMismatch, + EchoCapabilityContextMismatch, + LocalInitialContextMismatch, + LocalInitialNotSelected(BlockReference), + ConflictingHeaderContent(BlockReference), + ConflictingInitialHeader { + existing: BlockReference, + received: BlockReference, + }, + UnexpectedRecoveredHeader(BlockReference), HeaderUnavailable(BlockReference), WrongRecipient { expected: AuthorityIndex, @@ -246,6 +769,8 @@ pub(crate) enum RbcError { LoopbackPhase, InvalidPhaseTag, InvalidInitialTag, + InvalidInitialProof, + InitialProofSchemeMismatch, InitialSignatureRequiresSignatureAuthentication, InitialMacRequiresMacAuthentication, InitialAuthorMismatch { @@ -294,6 +819,93 @@ impl fmt::Display for RbcError { write!(f, "unknown Starfish-RBC authority {authority}") } Self::GenesisSlot => f.write_str("Starfish-RBC does not certify genesis slots"), + Self::FutureRound { round, maximum } => write!( + f, + "Starfish-RBC round {round} exceeds admission maximum {maximum}" + ), + Self::StaleRound { round, minimum } => write!( + f, + "Starfish-RBC round {round} is below admission minimum {minimum}" + ), + Self::RoundRegression { current, proposed } => write!( + f, + "Starfish-RBC local round cannot regress from {current} to {proposed}" + ), + Self::RetainedRoundRegression { current, proposed } => write!( + f, + "Starfish-RBC retained-round floor cannot regress from {current} to {proposed}" + ), + Self::RetainedRoundAheadOfLocal { local, proposed } => write!( + f, + "Starfish-RBC retained-round floor {proposed} exceeds local round {local}" + ), + Self::MissingAcknowledgments => { + f.write_str("Starfish-RBC header is missing acknowledgment fields") + } + Self::NonCanonicalAcknowledgments => { + f.write_str("Starfish-RBC acknowledgment encoding is not canonical") + } + Self::MissingTransactionsCommitment => { + f.write_str("Starfish-RBC header is missing its transaction commitment") + } + Self::ForbiddenHeaderExtensions => { + f.write_str("Starfish-RBC header carries a forbidden protocol extension") + } + Self::TooManyHeaderReferences { + field, + count, + maximum, + } => write!( + f, + "Starfish-RBC {field} reference count {count} exceeds limit {maximum}" + ), + Self::HeaderContentTooLarge => f.write_str("Starfish-RBC header content is too large"), + Self::ParentNotPast(parent) => { + write!(f, "Starfish-RBC parent {parent} is not from a past round") + } + Self::DuplicateParent(parent) => { + write!(f, "Starfish-RBC parent {parent} is duplicated") + } + Self::AcknowledgmentFromFuture(acknowledgment) => write!( + f, + "Starfish-RBC acknowledgment {acknowledgment} is from a future round" + ), + Self::DuplicateAcknowledgment(acknowledgment) => write!( + f, + "Starfish-RBC acknowledgment {acknowledgment} is duplicated" + ), + Self::InvalidThresholdClock => { + f.write_str("Starfish-RBC header does not reference previous-round quorum stake") + } + Self::HeaderDigestMismatch { expected, actual } => write!( + f, + "Starfish-RBC header digest mismatch: expected {expected}, got {actual}" + ), + Self::PinnedHeaderCommitteeMismatch => { + f.write_str("Starfish-RBC pinned header belongs to a different committee") + } + Self::EchoCapabilityContextMismatch => f.write_str( + "Starfish-RBC ECHO capability belongs to a different context or recipient", + ), + Self::LocalInitialContextMismatch => { + f.write_str("Starfish-RBC local initial handle belongs to a different kernel") + } + Self::LocalInitialNotSelected(block_ref) => write!( + f, + "Starfish-RBC local initial header {block_ref} is not the selected pinned proposal" + ), + Self::ConflictingHeaderContent(block_ref) => write!( + f, + "Starfish-RBC received conflicting pinned content for {block_ref}" + ), + Self::ConflictingInitialHeader { existing, received } => write!( + f, + "Starfish-RBC slot already staged initial header {existing}, not {received}" + ), + Self::UnexpectedRecoveredHeader(block_ref) => write!( + f, + "Starfish-RBC recovered header {block_ref} has no retained candidate" + ), Self::HeaderUnavailable(block_ref) => { write!( f, @@ -317,6 +929,12 @@ impl fmt::Display for RbcError { } Self::InvalidPhaseTag => f.write_str("Starfish-RBC phase MAC verification failed"), Self::InvalidInitialTag => f.write_str("Starfish-RBC initial MAC verification failed"), + Self::InvalidInitialProof => { + f.write_str("Starfish-RBC initial proof verification failed") + } + Self::InitialProofSchemeMismatch => { + f.write_str("Starfish-RBC initial proof has the wrong authentication scheme") + } Self::InitialSignatureRequiresSignatureAuthentication => f.write_str( "Starfish-RBC initial signature digest requires a signature authentication mode", ), @@ -334,7 +952,7 @@ impl fmt::Display for RbcError { impl Error for RbcError {} struct CandidateState { - header_available: bool, + header: Option, echoes: StakeAggregator, readies: StakeAggregator, echo_quorum_observed: bool, @@ -346,7 +964,7 @@ struct CandidateState { impl CandidateState { fn new() -> Self { Self { - header_available: false, + header: None, echoes: StakeAggregator::new(), readies: StakeAggregator::new(), echo_quorum_observed: false, @@ -367,14 +985,51 @@ impl CandidateState { } } -#[derive(Default)] struct SlotState { echoed: Option, readied: Option, delivered: Option, + initial_candidate: Option, + echo_by_sender: AHashMap, + ready_by_sender: AHashMap, candidates: AHashMap, } +impl Default for SlotState { + fn default() -> Self { + Self { + echoed: None, + readied: None, + delivered: None, + initial_candidate: None, + echo_by_sender: AHashMap::new(), + ready_by_sender: AHashMap::new(), + candidates: AHashMap::new(), + } + } +} + +impl SlotState { + fn record_phase_sender( + &mut self, + phase: RbcPhase, + sender: AuthorityIndex, + block_ref: BlockReference, + ) -> bool { + let seen = match phase { + RbcPhase::Echo => &mut self.echo_by_sender, + RbcPhase::Ready => &mut self.ready_by_sender, + }; + match seen.get(&sender) { + Some(existing) => *existing == block_ref, + None => { + seen.insert(sender, block_ref); + true + } + } + } +} + enum ProgressAction { NeedHeader(AuthoritySet), SendReady, @@ -387,6 +1042,8 @@ pub(crate) struct StarfishRbcKernel { own_authority: AuthorityIndex, context: RbcContext, mac_keys: Arc>, + local_round: RoundNumber, + minimum_new_slot_round: RoundNumber, slots: BTreeMap>, } @@ -397,6 +1054,7 @@ impl StarfishRbcKernel { protocol_instance: RbcProtocolInstanceId, initial_authentication: BlockAuthenticationScheme, mac_keys: Arc>, + local_round: RoundNumber, ) -> Result { let context = RbcContext::new(protocol_instance, &committee, initial_authentication)?; if !committee.known_authority(own_authority) { @@ -413,6 +1071,8 @@ impl StarfishRbcKernel { own_authority, context, mac_keys, + local_round, + minimum_new_slot_round: 1, slots: BTreeMap::new(), }) } @@ -421,47 +1081,287 @@ impl StarfishRbcKernel { self.context } - /// Record deterministic content validation and local header availability. - /// This does not authorize ECHO and does not imply initial authentication. - /// It remains module-private until header staging can supply a typed, - /// pinned content-validation result. - fn note_header_available( - &mut self, - block_ref: BlockReference, - ) -> Result, RbcError> { - self.validate_block_ref(&block_ref)?; - self.candidate_mut(block_ref).header_available = true; - Ok(self.drive(block_ref)) + pub(crate) fn maximum_admissible_round(&self) -> RoundNumber { + self.local_round.saturating_add(MAX_RBC_FUTURE_ROUNDS) } - /// Authorize the one local ECHO for this slot. It remains module-private - /// until the integration layer can pass a typed direct-author proof (or a - /// local-creation capability) instead of relying on call ordering. - fn authorize_echo(&mut self, block_ref: BlockReference) -> Result, RbcError> { - self.validate_block_ref(&block_ref)?; - let header_available = self - .candidate(&block_ref) - .is_some_and(|candidate| candidate.header_available); - if !header_available { - return Err(RbcError::HeaderUnavailable(block_ref)); + pub(crate) fn advance_local_round(&mut self, round: RoundNumber) -> Result<(), RbcError> { + if round < self.local_round { + return Err(RbcError::RoundRegression { + current: self.local_round, + proposed: round, + }); } + self.local_round = round; + Ok(()) + } - let own_authority = self.own_authority; - let committee = Arc::clone(&self.committee); - let slot = self.slot_mut(block_ref); - if slot.echoed.is_some() { - return Ok(Vec::new()); - } - slot.echoed = Some(block_ref); - slot.candidates - .entry(block_ref) - .or_insert_with(CandidateState::new) - .echoes - .add(own_authority, &committee); + pub(crate) fn minimum_new_slot_round(&self) -> RoundNumber { + self.minimum_new_slot_round + } - let mut effects = vec![RbcEffect::MulticastPhase { - phase: RbcPhase::Echo, - block_ref, + /// Reject allocation of previously unseen slots below a monotonic safe + /// watermark. Advancing the DAG round is not sufficient evidence for this + /// call: the integration layer may advance it only when its recovery model + /// proves that no newly observed slot below `round` is still required. + /// Existing slots remain active so late evidence can complete totality. + pub(crate) fn close_new_slots_before(&mut self, round: RoundNumber) -> Result<(), RbcError> { + if round < self.minimum_new_slot_round { + return Err(RbcError::RetainedRoundRegression { + current: self.minimum_new_slot_round, + proposed: round, + }); + } + let local_boundary = RoundNumber::max(self.local_round, 1); + if round > local_boundary { + return Err(RbcError::RetainedRoundAheadOfLocal { + local: self.local_round, + proposed: round, + }); + } + self.minimum_new_slot_round = round; + Ok(()) + } + + pub(crate) fn validate_header_content( + &self, + header: RbcCanonicalHeader, + ) -> Result { + self.validate_block_ref(&header.reference())?; + PinnedRbcHeader::validate_with_committee_id( + header, + &self.committee, + self.context.committee_id, + ) + } + + fn direct_initial_header( + &self, + direct_peer: AuthorityIndex, + header: PinnedRbcHeader, + proof: &RbcInitialProof, + ) -> Result { + header.ensure_committee(self.context.committee_id)?; + let block_ref = header.reference(); + self.validate_block_ref(&block_ref)?; + if direct_peer != block_ref.authority { + return Err(RbcError::InitialAuthorMismatch { + expected: block_ref.authority, + actual: direct_peer, + }); + } + if direct_peer == self.own_authority { + return Err(RbcError::LoopbackPhase); + } + + match (self.context.initial_authentication, proof) { + (BlockAuthenticationScheme::Ed25519, RbcInitialProof::Ed25519(signature)) => { + let digest = self.initial_signature_digest(block_ref)?; + let public_key = self + .committee + .get_public_key(block_ref.authority) + .ok_or(RbcError::UnknownAuthority(block_ref.authority))?; + public_key + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcError::InvalidInitialProof)?; + } + (BlockAuthenticationScheme::MlDsa44, RbcInitialProof::MlDsa44(signature)) => { + let digest = BlockDigest::from(self.initial_signature_digest(block_ref)?); + let public_key = self + .committee + .get_ml_dsa_44_public_key(block_ref.authority) + .ok_or(RbcError::UnknownAuthority(block_ref.authority))?; + public_key + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcError::InvalidInitialProof)?; + } + (BlockAuthenticationScheme::MlDsa65, RbcInitialProof::MlDsa65(signature)) => { + let digest = BlockDigest::from(self.initial_signature_digest(block_ref)?); + let public_key = self + .committee + .get_ml_dsa_65_public_key(block_ref.authority) + .ok_or(RbcError::UnknownAuthority(block_ref.authority))?; + public_key + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcError::InvalidInitialProof)?; + } + (BlockAuthenticationScheme::MacVector, RbcInitialProof::Mac(tag)) => { + self.verify_initial_mac_tag(direct_peer, block_ref, tag)?; + } + _ => return Err(RbcError::InitialProofSchemeMismatch), + } + Ok(EchoEligibleHeader { + header, + context: self.context, + recipient: self.own_authority, + }) + } + + fn local_initial_header( + &self, + header: PinnedRbcHeader, + ) -> Result { + header.ensure_committee(self.context.committee_id)?; + let block_ref = header.reference(); + self.validate_block_ref(&block_ref)?; + if block_ref.authority != self.own_authority { + return Err(RbcError::InitialAuthorMismatch { + expected: self.own_authority, + actual: block_ref.authority, + }); + } + Ok(EchoEligibleHeader { + header, + context: self.context, + recipient: self.own_authority, + }) + } + + /// Atomically construct, validate, select, pin, and ECHO a local-author + /// proposal before exposing it for authentication or dissemination. The + /// caller supplies no author or digest and cannot obtain two conflicting + /// local handles for one slot. + pub(crate) fn start_local_initial_header( + &mut self, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + ) -> Result { + let canonical = RbcCanonicalHeader::try_new( + self.own_authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions_commitment, + )?; + let pinned = self.validate_header_content(canonical)?; + let eligible = self.local_initial_header(pinned.clone())?; + let effects = self.accept_initial_header(eligible)?; + Ok(RbcLocalInitial { + header: pinned, + effects, + context: self.context, + author: self.own_authority, + }) + } + + fn accept_initial_header( + &mut self, + eligible: EchoEligibleHeader, + ) -> Result, RbcError> { + if eligible.context != self.context || eligible.recipient != self.own_authority { + return Err(RbcError::EchoCapabilityContextMismatch); + } + eligible + .header + .ensure_committee(self.context.committee_id)?; + let block_ref = eligible.header.reference(); + self.record_initial_candidate(block_ref)?; + let mut effects = self.note_header_available(eligible.header)?; + effects.extend(self.authorize_echo(block_ref)?); + Ok(effects) + } + + /// Validate and stage a directly received proposal before checking its + /// receiver-specific proof. Invalid authentication therefore cannot make + /// the adapter accidentally discard content needed by later READY + /// recovery. The outcome preserves any effects unblocked by staging. + pub(crate) fn accept_direct_initial_header( + &mut self, + direct_peer: AuthorityIndex, + header: RbcCanonicalHeader, + proof: &RbcInitialProof, + ) -> Result { + let pinned = self.validate_header_content(header)?; + let block_ref = pinned.reference(); + if direct_peer != block_ref.authority { + return Err(RbcError::InitialAuthorMismatch { + expected: block_ref.authority, + actual: direct_peer, + }); + } + if direct_peer == self.own_authority { + return Err(RbcError::LoopbackPhase); + } + self.record_initial_candidate(block_ref)?; + let mut effects = self.note_header_available(pinned.clone())?; + match self.direct_initial_header(direct_peer, pinned, proof) { + Ok(eligible) => { + effects.extend(self.accept_initial_header(eligible)?); + Ok(RbcInitialHeaderOutcome::Authenticated { effects }) + } + Err(error) => Ok(RbcInitialHeaderOutcome::StagedUnauthenticated { effects, error }), + } + } + + pub(crate) fn accept_recovered_header( + &mut self, + header: RbcCanonicalHeader, + ) -> Result, RbcError> { + let pinned = self.validate_header_content(header)?; + let block_ref = pinned.reference(); + self.validate_block_ref(&block_ref)?; + if self.candidate(&block_ref).is_none() { + return Err(RbcError::UnexpectedRecoveredHeader(block_ref)); + } + self.note_header_available(pinned) + } + + /// Record a pinned, deterministically content-validated header. This does + /// not authorize ECHO and does not imply initial authentication. External + /// ingress uses `accept_recovered_header` or an echo-eligible capability. + fn note_header_available( + &mut self, + header: PinnedRbcHeader, + ) -> Result, RbcError> { + header.ensure_committee(self.context.committee_id)?; + let block_ref = header.reference(); + self.validate_block_ref(&block_ref)?; + let candidate = self.candidate_mut(block_ref); + if candidate + .header + .as_ref() + .is_some_and(|existing| existing != &header) + { + return Err(RbcError::ConflictingHeaderContent(block_ref)); + } + candidate.header = Some(header); + Ok(self.drive(block_ref)) + } + + /// Complete the one local ECHO transition after the typed capability gate. + /// This lower-level method remains module-private so call ordering cannot + /// substitute for a direct-author proof or local-creation capability. + fn authorize_echo(&mut self, block_ref: BlockReference) -> Result, RbcError> { + self.validate_block_ref(&block_ref)?; + let header_available = self + .candidate(&block_ref) + .is_some_and(|candidate| candidate.header.is_some()); + if !header_available { + return Err(RbcError::HeaderUnavailable(block_ref)); + } + + let own_authority = self.own_authority; + let committee = Arc::clone(&self.committee); + let slot = self.slot_mut(block_ref); + if slot.echoed.is_some() { + return Ok(Vec::new()); + } + slot.echoed = Some(block_ref); + let recorded = slot.record_phase_sender(RbcPhase::Echo, own_authority, block_ref); + debug_assert!(recorded, "local ECHO guard and sender record diverged"); + slot.candidates + .entry(block_ref) + .or_insert_with(CandidateState::new) + .echoes + .add(own_authority, &committee); + + let mut effects = vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, }]; effects.extend(self.drive(block_ref)); Ok(effects) @@ -474,7 +1374,14 @@ impl StarfishRbcKernel { ) -> Result, RbcError> { self.verify_phase_message(direct_peer, &message)?; let committee = Arc::clone(&self.committee); - let candidate = self.candidate_mut(message.block_ref); + let slot = self.slot_mut(message.block_ref); + if !slot.record_phase_sender(message.phase, message.sender, message.block_ref) { + return Ok(Vec::new()); + } + let candidate = slot + .candidates + .entry(message.block_ref) + .or_insert_with(CandidateState::new); match message.phase { RbcPhase::Echo => { candidate.echoes.add(message.sender, &committee); @@ -504,7 +1411,7 @@ impl StarfishRbcKernel { && slot .candidates .get(&block_ref) - .is_some_and(|candidate| candidate.header_available) + .is_some_and(|candidate| candidate.header.is_some()) }); if !authorized { return Err(RbcError::PhaseNotAuthorized { phase, block_ref }); @@ -532,12 +1439,38 @@ impl StarfishRbcKernel { }) } - /// Produce the common 32-byte digest signed by Ed25519 or ML-DSA for an - /// initial Starfish-RBC header proposal. - pub(crate) fn initial_signature_digest( + fn ensure_local_initial(&self, local: &RbcLocalInitial) -> Result { + if local.context != self.context || local.author != self.own_authority { + return Err(RbcError::LocalInitialContextMismatch); + } + local.header.ensure_committee(self.context.committee_id)?; + let block_ref = local.header.reference(); + let selected = self.slot(&block_ref).is_some_and(|slot| { + slot.initial_candidate == Some(block_ref) + && slot.echoed == Some(block_ref) + && slot + .candidates + .get(&block_ref) + .and_then(|candidate| candidate.header.as_ref()) + == Some(&local.header) + }); + if !selected { + return Err(RbcError::LocalInitialNotSelected(block_ref)); + } + Ok(block_ref) + } + + pub(crate) fn make_local_initial_signature_digest( &self, - block_ref: BlockReference, + local: &RbcLocalInitial, ) -> Result<[u8; 32], RbcError> { + let block_ref = self.ensure_local_initial(local)?; + self.initial_signature_digest(block_ref) + } + + /// Produce the common 32-byte digest signed by Ed25519 or ML-DSA for an + /// initial Starfish-RBC header proposal. + fn initial_signature_digest(&self, block_ref: BlockReference) -> Result<[u8; 32], RbcError> { if self.context.initial_authentication == BlockAuthenticationScheme::MacVector { return Err(RbcError::InitialSignatureRequiresSignatureAuthentication); } @@ -548,7 +1481,16 @@ impl StarfishRbcKernel { /// Produce one receiver-specific initial MAC. The local author calls this /// separately for every non-local recipient. - pub(crate) fn make_initial_mac_tag( + pub(crate) fn make_local_initial_mac_tag( + &self, + local: &RbcLocalInitial, + recipient: AuthorityIndex, + ) -> Result { + let block_ref = self.ensure_local_initial(local)?; + self.make_initial_mac_tag_for_reference(block_ref, recipient) + } + + fn make_initial_mac_tag_for_reference( &self, block_ref: BlockReference, recipient: AuthorityIndex, @@ -634,7 +1576,7 @@ impl StarfishRbcKernel { return Ok(None); }; let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; - let blocked_on_header = !candidate.header_available + let blocked_on_header = candidate.header.is_none() && ((slot.readied.is_none() && ready_trigger) || (slot.delivered.is_none() && candidate.ready_quorum_observed)); Ok(blocked_on_header.then(|| RbcEffect::NeedHeader { @@ -694,9 +1636,38 @@ impl StarfishRbcKernel { if !self.committee.known_authority(block_ref.authority) { return Err(RbcError::UnknownAuthority(block_ref.authority)); } + let maximum_round = self.maximum_admissible_round(); + if block_ref.round > maximum_round { + return Err(RbcError::FutureRound { + round: block_ref.round, + maximum: maximum_round, + }); + } + if block_ref.round < self.minimum_new_slot_round && self.slot(block_ref).is_none() { + return Err(RbcError::StaleRound { + round: block_ref.round, + minimum: self.minimum_new_slot_round, + }); + } Ok(()) } + fn record_initial_candidate(&mut self, block_ref: BlockReference) -> Result<(), RbcError> { + self.validate_block_ref(&block_ref)?; + let slot = self.slot_mut(block_ref); + match slot.initial_candidate { + Some(existing) if existing != block_ref => Err(RbcError::ConflictingInitialHeader { + existing, + received: block_ref, + }), + Some(_) => Ok(()), + None => { + slot.initial_candidate = Some(block_ref); + Ok(()) + } + } + } + fn slot_mut(&mut self, block_ref: BlockReference) -> &mut SlotState { self.slots .entry(block_ref.round) @@ -741,16 +1712,16 @@ impl StarfishRbcKernel { let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; - let blocked_on_header = !candidate.header_available + let blocked_on_header = candidate.header.is_none() && ((can_send_ready && ready_trigger) || (can_deliver && candidate.ready_quorum_observed)); let holders = candidate.holders(); if blocked_on_header && holders != candidate.header_request_holders { candidate.header_request_holders = holders; ProgressAction::NeedHeader(holders) - } else if candidate.header_available && can_send_ready && ready_trigger { + } else if candidate.header.is_some() && can_send_ready && ready_trigger { ProgressAction::SendReady - } else if candidate.header_available + } else if candidate.header.is_some() && can_deliver && candidate.ready_quorum_observed { @@ -771,6 +1742,9 @@ impl StarfishRbcKernel { let slot = self.slot_mut(block_ref); if slot.readied.is_none() { slot.readied = Some(block_ref); + let recorded = + slot.record_phase_sender(RbcPhase::Ready, own_authority, block_ref); + debug_assert!(recorded, "local READY guard and sender record diverged"); slot.candidates .entry(block_ref) .or_insert_with(CandidateState::new) @@ -786,7 +1760,12 @@ impl StarfishRbcKernel { let slot = self.slot_mut(block_ref); if slot.delivered.is_none() { slot.delivered = Some(block_ref); - effects.push(RbcEffect::Deliver(block_ref)); + let header = slot + .candidates + .get(&block_ref) + .and_then(|candidate| candidate.header.clone()) + .expect("delivery requires a pinned Starfish-RBC header"); + effects.push(RbcEffect::Deliver(header)); } } ProgressAction::None => break, @@ -897,11 +1876,15 @@ mod tests { use super::*; use crate::{ - crypto::mac_keyrings_for_test, + crypto::{ + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + }, types::{BlockDigest, BlockReference}, }; const TEST_INSTANCE_BYTE: u8 = 0xA5; + type DeliveryTrace = Vec>; + type RecoveryTrace = Vec<(AuthorityIndex, AuthorityIndex, BlockReference)>; fn block(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { BlockReference { @@ -911,6 +1894,81 @@ mod tests { } } + fn pinned_header_for_context( + context: RbcContext, + block_ref: BlockReference, + ) -> PinnedRbcHeader { + PinnedRbcHeader { + header: Arc::new(RbcCanonicalHeader { + reference: block_ref, + block_references: Vec::new(), + acknowledgments: RbcAckFields { + intersection: Some(0), + extra_references: Vec::new(), + }, + meta_creation_time_ns: 0, + transactions_commitment: TransactionsCommitment::default(), + }), + committee_id: context.committee_id, + } + } + + fn pinned_header(committee: &Committee, block_ref: BlockReference) -> PinnedRbcHeader { + pinned_header_for_context( + RbcContext::new( + instance(TEST_INSTANCE_BYTE), + committee, + BlockAuthenticationScheme::Ed25519, + ) + .unwrap(), + block_ref, + ) + } + + fn valid_canonical_header( + authority: AuthorityIndex, + round: RoundNumber, + marker: u8, + ) -> RbcCanonicalHeader { + let parents = (0..3) + .map(|parent_authority| { + block( + parent_authority, + round - 1, + marker.wrapping_add(parent_authority as u8), + ) + }) + .collect(); + RbcCanonicalHeader::try_new( + authority, + round, + parents, + Vec::new(), + 0x0102_0304_0506_0708, + TransactionsCommitment::default(), + ) + .unwrap() + } + + fn block_header_from_canonical( + header: &RbcCanonicalHeader, + authentication: BlockAuthentication, + ) -> BlockHeader { + BlockHeader { + reference: header.reference, + block_references: header.block_references.clone(), + meta_creation_time_ns: header.meta_creation_time_ns, + authentication, + transactions_commitment: Some(header.transactions_commitment), + ack: Some(header.acknowledgment_fields()), + strong_vote: None, + bls: None, + sailfish: None, + unprovable_certificate: None, + serialized: None, + } + } + fn instance(marker: u8) -> RbcProtocolInstanceId { RbcProtocolInstanceId::new([marker; 32]).unwrap() } @@ -943,6 +2001,7 @@ mod tests { instance(instance_byte), authentication, Arc::new(keyrings[own_authority as usize].clone()), + 0, ) .unwrap() } @@ -978,7 +2037,8 @@ mod tests { } fn authorize_echo(kernel: &mut StarfishRbcKernel, block_ref: BlockReference) { - kernel.note_header_available(block_ref).unwrap(); + let header = pinned_header_for_context(kernel.context(), block_ref); + kernel.note_header_available(header).unwrap(); assert!(matches!( kernel.authorize_echo(block_ref).unwrap().as_slice(), [RbcEffect::MulticastPhase { @@ -995,8 +2055,8 @@ mod tests { fn pump_phase_effects( kernels: &mut [StarfishRbcKernel], initial_effects: Vec<(AuthorityIndex, Vec)>, - fetch_missing_headers: bool, - ) -> Vec> { + header_stores: &mut [AHashMap], + ) -> (DeliveryTrace, RecoveryTrace) { let mut queue: VecDeque<_> = initial_effects .into_iter() .flat_map(|(authority, effects)| { @@ -1004,6 +2064,7 @@ mod tests { }) .collect(); let mut deliveries = vec![Vec::new(); kernels.len()]; + let mut recoveries = Vec::new(); while let Some((owner, effect)) = queue.pop_front() { match effect { @@ -1027,46 +2088,886 @@ mod tests { queue.extend(effects.into_iter().map(|effect| (recipient, effect))); } } - RbcEffect::NeedHeader { block_ref, .. } if fetch_missing_headers => { + RbcEffect::NeedHeader { block_ref, holders } => { + let (source, header) = holders + .present() + .find_map(|source| { + header_stores[source as usize] + .get(&block_ref) + .cloned() + .map(|header| (source, header)) + }) + .expect("a test RBC holder must retain the canonical header"); let effects = kernels[owner as usize] - .note_header_available(block_ref) + .accept_recovered_header(header.clone()) .unwrap(); + header_stores[owner as usize].insert(block_ref, header); + recoveries.push((owner, source, block_ref)); queue.extend(effects.into_iter().map(|effect| (owner, effect))); } - RbcEffect::NeedHeader { .. } => {} - RbcEffect::Deliver(block_ref) => { - deliveries[owner as usize].push(block_ref); + RbcEffect::Deliver(header) => { + assert_eq!( + header_stores[owner as usize].get(&header.reference()), + Some(header.header()) + ); + deliveries[owner as usize].push(header.reference()); } } } - deliveries + (deliveries, recoveries) + } + + #[test] + fn canonical_statement_layout_is_fixed_width_and_big_endian() { + let context = RbcContext { + protocol_instance: RbcProtocolInstanceId([0x11; 32]), + committee_id: RbcCommitteeId([0x22; 32]), + initial_authentication: BlockAuthenticationScheme::MlDsa65, + }; + let block_ref = BlockReference { + authority: 0x0102, + round: 0x0304_0506, + digest: BlockDigest::from([0x33; 32]), + }; + let statement = encode_mac_statement(&context, ECHO_KIND, &block_ref, 0x0708, 0x090A); + + assert_eq!(statement.len(), 123); + assert_eq!(&statement[..15], PROTOCOL_DOMAIN); + assert_eq!(statement[15], ECHO_KIND); + assert_eq!(statement[16], 0x02); + assert_eq!(&statement[17..49], &[0x11; 32]); + assert_eq!(&statement[49..81], &[0x22; 32]); + assert_eq!(&statement[81..83], &[0x01, 0x02]); + assert_eq!(&statement[83..87], &[0x03, 0x04, 0x05, 0x06]); + assert_eq!(&statement[87..119], &[0x33; 32]); + assert_eq!(&statement[119..121], &[0x07, 0x08]); + assert_eq!(&statement[121..123], &[0x09, 0x0A]); + } + + #[test] + fn canonical_header_digest_has_a_frozen_tagged_encoding() { + let parent = block(0x0708, 0x090A_0B0C, 0x11); + let acknowledgment = block(0x0D0E, 0x0F10_1112, 0x22); + let timestamp: TimestampNs = 0x1314_1516_1718_191A; + let commitment = TransactionsCommitment::default(); + + let mut encoded = Vec::new(); + encoded.extend_from_slice(&[0x01]); + encoded.extend_from_slice(&0x0102u16.to_be_bytes()); + encoded.extend_from_slice(&[0x02]); + encoded.extend_from_slice(&0x0304_0506u32.to_be_bytes()); + encoded.extend_from_slice(&[0x03]); + encoded.extend_from_slice(&1u32.to_be_bytes()); + encoded.extend_from_slice(&parent.authority.to_be_bytes()); + encoded.extend_from_slice(&parent.round.to_be_bytes()); + encoded.extend_from_slice(parent.digest.as_ref()); + encoded.extend_from_slice(&[0x04]); + encoded.extend_from_slice(&1u32.to_be_bytes()); + encoded.extend_from_slice(&acknowledgment.authority.to_be_bytes()); + encoded.extend_from_slice(&acknowledgment.round.to_be_bytes()); + encoded.extend_from_slice(acknowledgment.digest.as_ref()); + encoded.extend_from_slice(&[0x05]); + encoded.extend_from_slice(×tamp.to_be_bytes()); + encoded.extend_from_slice(&[0x06]); + encoded.extend_from_slice(commitment.as_ref()); + + assert_eq!( + hex::encode(&encoded), + concat!( + "010102020304050603000000010708090a0b0c", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "04000000010d0e0f101112", + "22222222222222222222222222222222", + "22222222222222222222222222222222", + "051314", + "15161718191a060000000000000000000000000000000000000000000000000000", + "000000000000" + ) + ); + let digest = BlockDigest::new_starfish_rbc_header( + 0x0102, + 0x0304_0506, + &[parent], + &[acknowledgment], + timestamp, + commitment, + ); + assert_eq!(digest.as_ref(), blake3::hash(&encoded).as_bytes()); + assert_eq!( + hex::encode(digest.as_ref()), + "3a0ef697511a95ddf97c73e72aad2cb2313839063f7249fe0f967f2d8ff3ad22" + ); + } + + #[test] + fn canonical_digest_separates_the_legacy_parent_ack_boundary() { + let references: Vec<_> = (0..4) + .map(|authority| block(authority, 7, 0x30 + authority as u8)) + .collect(); + let commitment = TransactionsCommitment::default(); + let legacy_first = BlockDigest::new( + 0, + 8, + &references[..3], + &references[3..], + 42, + Some(commitment), + None, + ); + let legacy_second = BlockDigest::new(0, 8, &references, &[], 42, Some(commitment), None); + assert_eq!(legacy_first, legacy_second); + + let canonical_first = BlockDigest::new_starfish_rbc_header( + 0, + 8, + &references[..3], + &references[3..], + 42, + commitment, + ); + let canonical_second = + BlockDigest::new_starfish_rbc_header(0, 8, &references, &[], 42, commitment); + assert_ne!(canonical_first, canonical_second); + } + + #[test] + fn canonical_digest_binds_every_content_field_and_order() { + let first = block(0, 4, 0x41); + let second = block(1, 4, 0x42); + let commitment = TransactionsCommitment::default(); + let other_commitment = TransactionsCommitment::new_from_transactions(&Vec::new()); + let digest = + BlockDigest::new_starfish_rbc_header(2, 5, &[first, second], &[first], 7, commitment); + + for changed in [ + BlockDigest::new_starfish_rbc_header(3, 5, &[first, second], &[first], 7, commitment), + BlockDigest::new_starfish_rbc_header(2, 6, &[first, second], &[first], 7, commitment), + BlockDigest::new_starfish_rbc_header(2, 5, &[second, first], &[first], 7, commitment), + BlockDigest::new_starfish_rbc_header(2, 5, &[first, second], &[second], 7, commitment), + BlockDigest::new_starfish_rbc_header(2, 5, &[first, second], &[first], 8, commitment), + BlockDigest::new_starfish_rbc_header( + 2, + 5, + &[first, second], + &[first], + 7, + other_commitment, + ), + ] { + assert_ne!(digest, changed); + } + } + + #[test] + fn canonical_header_validation_is_authentication_independent_and_pins_content() { + let committee = Committee::new_test(vec![1; 4]); + let canonical = valid_canonical_header(3, 5, 0x51); + let without_authentication = + block_header_from_canonical(&canonical, BlockAuthentication::None); + let with_authentication = block_header_from_canonical( + &canonical, + BlockAuthentication::Ed25519(SignatureBytes::default()), + ); + + let extracted_without = + RbcCanonicalHeader::from_block_header(&without_authentication).unwrap(); + let extracted_with = RbcCanonicalHeader::from_block_header(&with_authentication).unwrap(); + assert_eq!(extracted_without, canonical); + assert_eq!(extracted_with, canonical); + + let pinned = PinnedRbcHeader::validate(canonical.clone(), &committee).unwrap(); + let retained = pinned.clone(); + assert_eq!(pinned.reference(), canonical.reference()); + assert!(Arc::ptr_eq(&pinned.header, &retained.header)); + assert_eq!(pinned.header(), &canonical); + } + + #[test] + fn block_header_conversion_rejects_missing_fields_and_extensions() { + let canonical = valid_canonical_header(3, 5, 0x52); + let mut header = block_header_from_canonical(&canonical, BlockAuthentication::None); + header.ack = None; + assert_eq!( + RbcCanonicalHeader::from_block_header(&header), + Err(RbcError::MissingAcknowledgments) + ); + + let mut header = block_header_from_canonical(&canonical, BlockAuthentication::None); + header.transactions_commitment = None; + assert_eq!( + RbcCanonicalHeader::from_block_header(&header), + Err(RbcError::MissingTransactionsCommitment) + ); + + let mut header = block_header_from_canonical(&canonical, BlockAuthentication::None); + header.strong_vote = Some(AuthoritySet::default()); + assert_eq!( + RbcCanonicalHeader::from_block_header(&header), + Err(RbcError::ForbiddenHeaderExtensions) + ); + + let mut header = block_header_from_canonical(&canonical, BlockAuthentication::None); + header.block_references = vec![block(0, 4, 0x51); MAX_RBC_REFERENCES_PER_FIELD + 1]; + assert!(matches!( + RbcCanonicalHeader::from_block_header(&header), + Err(RbcError::TooManyHeaderReferences { + field: "parent", + .. + }) + )); + + let mut header = block_header_from_canonical(&canonical, BlockAuthentication::None); + header.ack = Some(AckFields { + intersection: None, + extra_references: vec![block(0, 4, 0x51); MAX_RBC_REFERENCES_PER_FIELD + 1], + }); + assert!(matches!( + RbcCanonicalHeader::from_block_header(&header), + Err(RbcError::TooManyHeaderReferences { + field: "acknowledgment", + .. + }) + )); + + let mut header = block_header_from_canonical(&canonical, BlockAuthentication::None); + header.block_references = vec![block(0, 4, 0x51); MAX_RBC_REFERENCES_PER_FIELD]; + header.ack = Some(AckFields { + intersection: Some(0), + extra_references: vec![block(1, 4, 0x52)], + }); + assert!(matches!( + RbcCanonicalHeader::from_block_header(&header), + Err(RbcError::TooManyHeaderReferences { + field: "acknowledgment", + .. + }) + )); + } + + #[test] + fn acknowledgment_compression_is_canonical_and_preserves_u8_boundary() { + let committee = Committee::new_test(vec![1; 4]); + let canonical = valid_canonical_header(3, 5, 0x53); + assert!(PinnedRbcHeader::validate(canonical.clone(), &committee).is_ok()); + + let mut legacy_alias = canonical.clone(); + legacy_alias.acknowledgments.intersection = None; + legacy_alias.acknowledgments.extra_references.clear(); + assert_eq!( + PinnedRbcHeader::validate(legacy_alias, &committee), + Err(RbcError::NonCanonicalAcknowledgments) + ); + + let mut out_of_range = canonical; + out_of_range.acknowledgments.intersection = Some(4); + assert_eq!( + PinnedRbcHeader::validate(out_of_range, &committee), + Err(RbcError::NonCanonicalAcknowledgments) + ); + + let references = |count: usize| { + (0..count) + .map(|index| { + let mut digest = [0; 32]; + digest[..4].copy_from_slice(&(index as u32).to_be_bytes()); + BlockReference { + authority: index as AuthorityIndex % 4, + round: 4, + digest: BlockDigest::from(digest), + } + }) + .collect::>() + }; + let parents_255 = references(255); + let parents_256 = references(256); + assert_eq!( + RbcAckFields::from_logical(&parents_255, &[]).intersection, + Some(255) + ); + assert_eq!( + RbcAckFields::from_logical(&parents_256, &[]), + RbcAckFields { + intersection: None, + extra_references: Vec::new(), + } + ); + } + + #[test] + fn canonical_header_wire_keeps_starfish_acknowledgment_compression() { + #[derive(Serialize)] + struct ExpandedHeader<'a> { + reference: BlockReference, + block_references: &'a [BlockReference], + acknowledgments: &'a [BlockReference], + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + } + + let parents: Vec<_> = (0..100) + .map(|index| { + let mut digest = [0; 32]; + digest[..4].copy_from_slice(&(index as u32).to_be_bytes()); + BlockReference { + authority: index % 4, + round: 4, + digest: BlockDigest::from(digest), + } + }) + .collect(); + let logical_acknowledgments = parents[50..].to_vec(); + let header = RbcCanonicalHeader::try_new( + 3, + 5, + parents, + logical_acknowledgments.clone(), + 7, + TransactionsCommitment::default(), + ) + .unwrap(); + assert_eq!(header.acknowledgments.intersection, Some(50)); + assert!(header.acknowledgments.extra_references.is_empty()); + + let compressed_size = bincode::serialize(&header).unwrap().len(); + let expanded_size = bincode::serialize(&ExpandedHeader { + reference: header.reference, + block_references: &header.block_references, + acknowledgments: &logical_acknowledgments, + meta_creation_time_ns: header.meta_creation_time_ns, + transactions_commitment: header.transactions_commitment, + }) + .unwrap() + .len(); + assert!(compressed_size < expanded_size); + } + + #[test] + fn canonical_header_validation_rejects_duplicate_and_invalid_references() { + let committee = Committee::new_test(vec![1; 4]); + let valid = valid_canonical_header(3, 5, 0x54); + + let same_round_ack = RbcCanonicalHeader::try_new( + 3, + 5, + valid.block_references.clone(), + vec![block(2, 5, 0x59)], + valid.meta_creation_time_ns, + valid.transactions_commitment, + ) + .unwrap(); + assert!(PinnedRbcHeader::validate(same_round_ack, &committee).is_ok()); + + let mut duplicate_parent = valid.clone(); + duplicate_parent + .block_references + .push(duplicate_parent.block_references[0]); + duplicate_parent.reference.digest = BlockDigest::new_starfish_rbc_header( + duplicate_parent.reference.authority, + duplicate_parent.reference.round, + &duplicate_parent.block_references, + &duplicate_parent.acknowledgment_references(), + duplicate_parent.meta_creation_time_ns, + duplicate_parent.transactions_commitment, + ); + assert!(matches!( + PinnedRbcHeader::validate(duplicate_parent, &committee), + Err(RbcError::DuplicateParent(_)) + )); + + let duplicate_ack = block(3, 5, 0x55); + assert!(matches!( + RbcCanonicalHeader::try_new( + 3, + 5, + valid.block_references.clone(), + vec![duplicate_ack, duplicate_ack], + valid.meta_creation_time_ns, + valid.transactions_commitment, + ), + Err(RbcError::DuplicateAcknowledgment(_)) + )); + + let shared_parent = valid.block_references[2]; + let extra = block(3, 5, 0x5A); + let normalized = RbcCanonicalHeader::try_new( + 3, + 5, + valid.block_references.clone(), + vec![extra, shared_parent], + valid.meta_creation_time_ns, + valid.transactions_commitment, + ); + assert_eq!( + normalized.unwrap().acknowledgment_references(), + vec![shared_parent, extra] + ); + + let future_ack = block(3, 6, 0x56); + let future_ack_header = RbcCanonicalHeader::try_new( + 3, + 5, + valid.block_references.clone(), + vec![future_ack], + valid.meta_creation_time_ns, + valid.transactions_commitment, + ) + .unwrap(); + assert!(matches!( + PinnedRbcHeader::validate(future_ack_header, &committee), + Err(RbcError::AcknowledgmentFromFuture(_)) + )); + + let unknown_ack_header = RbcCanonicalHeader::try_new( + 3, + 5, + valid.block_references.clone(), + vec![block(4, 5, 0x5B)], + valid.meta_creation_time_ns, + valid.transactions_commitment, + ) + .unwrap(); + assert_eq!( + PinnedRbcHeader::validate(unknown_ack_header, &committee), + Err(RbcError::UnknownAuthority(4)) + ); + + let mut unknown_parent = valid.clone(); + unknown_parent.block_references[0].authority = 4; + unknown_parent.reference.digest = BlockDigest::new_starfish_rbc_header( + unknown_parent.reference.authority, + unknown_parent.reference.round, + &unknown_parent.block_references, + &unknown_parent.acknowledgment_references(), + unknown_parent.meta_creation_time_ns, + unknown_parent.transactions_commitment, + ); + assert_eq!( + PinnedRbcHeader::validate(unknown_parent, &committee), + Err(RbcError::UnknownAuthority(4)) + ); + + let mut same_round_parent = valid.clone(); + same_round_parent.block_references[0].round = same_round_parent.reference.round; + same_round_parent.reference.digest = BlockDigest::new_starfish_rbc_header( + same_round_parent.reference.authority, + same_round_parent.reference.round, + &same_round_parent.block_references, + &same_round_parent.acknowledgment_references(), + same_round_parent.meta_creation_time_ns, + same_round_parent.transactions_commitment, + ); + assert!(matches!( + PinnedRbcHeader::validate(same_round_parent, &committee), + Err(RbcError::ParentNotPast(_)) + )); + + let insufficient_parents = RbcCanonicalHeader::try_new( + 3, + 5, + valid.block_references[..2].to_vec(), + Vec::new(), + valid.meta_creation_time_ns, + valid.transactions_commitment, + ) + .unwrap(); + assert_eq!( + PinnedRbcHeader::validate(insufficient_parents, &committee), + Err(RbcError::InvalidThresholdClock) + ); + + let mut wrong_digest = valid; + wrong_digest.reference.digest = BlockDigest::from([0xFF; 32]); + assert!(matches!( + PinnedRbcHeader::validate(wrong_digest, &committee), + Err(RbcError::HeaderDigestMismatch { .. }) + )); + + let keyrings = mac_keyrings_for_test(4); + let kernel = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + let mut genesis = valid_canonical_header(0, 5, 0x5C); + genesis.reference.round = 0; + assert_eq!( + kernel.validate_header_content(genesis), + Err(RbcError::GenesisSlot) + ); + } + + #[test] + fn future_round_admission_advances_monotonically() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut kernel = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + let too_far = block(0, MAX_RBC_FUTURE_ROUNDS + 1, 0x57); + assert!(matches!( + kernel.handle_phase( + 1, + phase_message( + Arc::clone(&kernel.committee), + &keyrings, + 1, + 0, + RbcPhase::Echo, + too_far, + ), + ), + Err(RbcError::FutureRound { .. }) + )); + assert!(kernel.slots.is_empty()); + + kernel.advance_local_round(1).unwrap(); + assert_eq!(kernel.maximum_admissible_round(), MAX_RBC_FUTURE_ROUNDS + 1); + assert_eq!( + kernel.advance_local_round(0), + Err(RbcError::RoundRegression { + current: 1, + proposed: 0, + }) + ); + } + + #[test] + fn retained_round_floor_rejects_only_unseen_slots() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + let first = valid_canonical_header(0, 5, 0xD0).reference(); + receiver + .handle_phase( + 0, + phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Echo, + first, + ), + ) + .unwrap(); + + receiver.advance_local_round(10).unwrap(); + assert_eq!( + receiver.close_new_slots_before(11), + Err(RbcError::RetainedRoundAheadOfLocal { + local: 10, + proposed: 11, + }) + ); + receiver.close_new_slots_before(10).unwrap(); + assert_eq!(receiver.minimum_new_slot_round(), 10); + assert_eq!( + receiver.close_new_slots_before(9), + Err(RbcError::RetainedRoundRegression { + current: 10, + proposed: 9, + }) + ); + + let retained_candidate = valid_canonical_header(0, 5, 0xD1); + let retained_ref = retained_candidate.reference(); + receiver + .handle_phase( + 1, + phase_message( + Arc::clone(&committee), + &keyrings, + 1, + 3, + RbcPhase::Ready, + retained_ref, + ), + ) + .unwrap(); + assert!( + receiver + .accept_recovered_header(retained_candidate) + .unwrap() + .is_empty() + ); + + let unseen = block(1, 5, 0xD2); + assert_eq!( + receiver.handle_phase( + 0, + phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Echo, + unseen, + ), + ), + Err(RbcError::StaleRound { + round: 5, + minimum: 10, + }) + ); + assert!(receiver.slots[&5].get(&1).is_none()); + assert!(receiver.candidate(&retained_ref).unwrap().header.is_some()); + } + + #[test] + fn phase_sender_admission_bounds_equivocation_without_burning_invalid_messages() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let canonical = valid_canonical_header(0, 6, 0xD3); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + let outcome = receiver + .accept_direct_initial_header( + 0, + canonical, + &RbcInitialProof::Ed25519(SignatureBytes::default()), + ) + .unwrap(); + assert!(matches!( + outcome, + RbcInitialHeaderOutcome::StagedUnauthenticated { .. } + )); + let conflicting_initial = valid_canonical_header(0, 6, 0xD5); + let conflicting_ref = conflicting_initial.reference(); + assert!(matches!( + receiver.accept_direct_initial_header( + 0, + conflicting_initial, + &RbcInitialProof::Ed25519(SignatureBytes::default()), + ), + Err(RbcError::ConflictingInitialHeader { received, .. }) if received == conflicting_ref + )); + + let mut invalid = phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + RbcPhase::Echo, + block(0, 6, 0xD4), + ); + invalid.tag = MacTag::from_bytes([0; 32]); + assert_eq!( + receiver.handle_phase(0, invalid), + Err(RbcError::InvalidPhaseTag) + ); + + let mut first_echo = None; + let mut first_ready = None; + for sender in 0..3 { + let echo = block(0, 6, 0xE0 + sender as u8); + let ready = block(0, 6, 0xF0 + sender as u8); + first_echo.get_or_insert(echo); + first_ready.get_or_insert(ready); + receiver + .handle_phase( + sender, + phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Echo, + echo, + ), + ) + .unwrap(); + receiver + .handle_phase( + sender, + phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Ready, + ready, + ), + ) + .unwrap(); + } + + for marker in 0..32 { + for phase in [RbcPhase::Echo, RbcPhase::Ready] { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + 0, + 3, + phase, + block(0, 6, marker), + ); + assert!(receiver.handle_phase(0, message).unwrap().is_empty()); + } + } + let slot = receiver.slot(&block(0, 6, 0)).unwrap(); + assert_eq!(slot.candidates.len(), 1 + 2 * (committee.len() - 1)); + assert!(slot.candidates.contains_key(&first_echo.unwrap())); + assert!(slot.candidates.contains_key(&first_ready.unwrap())); + } + + #[test] + fn recovered_headers_require_prior_authenticated_phase_evidence() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let canonical = valid_canonical_header(0, 8, 0xD6); + let block_ref = canonical.reference(); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + + assert_eq!( + receiver.accept_recovered_header(canonical.clone()), + Err(RbcError::UnexpectedRecoveredHeader(block_ref)) + ); + assert!(receiver.slots.is_empty()); + + receiver + .handle_phase( + 0, + phase_message(committee, &keyrings, 0, 3, RbcPhase::Ready, block_ref), + ) + .unwrap(); + assert!( + receiver + .accept_recovered_header(canonical) + .unwrap() + .is_empty() + ); + assert!(receiver.candidate(&block_ref).unwrap().header.is_some()); } #[test] - fn canonical_statement_layout_is_fixed_width_and_big_endian() { - let context = RbcContext { - protocol_instance: RbcProtocolInstanceId([0x11; 32]), - committee_id: RbcCommitteeId([0x22; 32]), - initial_authentication: BlockAuthenticationScheme::MlDsa65, + fn bounded_header_decoder_rejects_oversized_reference_vector() { + #[derive(Serialize)] + struct UnboundedReferences { + references: Vec, + } + #[derive(Deserialize)] + struct BoundedReferences { + #[serde(with = "super::bounded_references")] + _references: Vec, + } + + let oversized = UnboundedReferences { + references: vec![block(0, 1, 0x58); MAX_RBC_REFERENCES_PER_FIELD + 1], }; - let block_ref = BlockReference { - authority: 0x0102, - round: 0x0304_0506, - digest: BlockDigest::from([0x33; 32]), + let bytes = bincode::serialize(&oversized).unwrap(); + assert!(bincode::deserialize::(&bytes).is_err()); + + #[derive(Serialize)] + struct UnboundedAckFields { + intersection: Option, + extra_references: Vec, + } + #[derive(Serialize)] + struct UnboundedHeader { + reference: BlockReference, + block_references: Vec, + acknowledgments: UnboundedAckFields, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + } + + let base = valid_canonical_header(0, 5, 0x59); + let oversized_parent_header = UnboundedHeader { + reference: base.reference, + block_references: vec![block(0, 4, 0x58); MAX_RBC_REFERENCES_PER_FIELD + 1], + acknowledgments: UnboundedAckFields { + intersection: Some(0), + extra_references: Vec::new(), + }, + meta_creation_time_ns: base.meta_creation_time_ns, + transactions_commitment: base.transactions_commitment, }; - let statement = encode_mac_statement(&context, ECHO_KIND, &block_ref, 0x0708, 0x090A); + let bytes = bincode::serialize(&oversized_parent_header).unwrap(); + assert!(bincode::deserialize::(&bytes).is_err()); + + let oversized_extra_header = UnboundedHeader { + reference: base.reference, + block_references: base.block_references, + acknowledgments: UnboundedAckFields { + intersection: Some(0), + extra_references: vec![block(0, 4, 0x58); MAX_RBC_REFERENCES_PER_FIELD + 1], + }, + meta_creation_time_ns: base.meta_creation_time_ns, + transactions_commitment: base.transactions_commitment, + }; + let bytes = bincode::serialize(&oversized_extra_header).unwrap(); + assert!(bincode::deserialize::(&bytes).is_err()); + } - assert_eq!(statement.len(), 123); - assert_eq!(&statement[..15], PROTOCOL_DOMAIN); - assert_eq!(statement[15], ECHO_KIND); - assert_eq!(statement[16], 0x02); - assert_eq!(&statement[17..49], &[0x11; 32]); - assert_eq!(&statement[49..81], &[0x22; 32]); - assert_eq!(&statement[81..83], &[0x01, 0x02]); - assert_eq!(&statement[83..87], &[0x03, 0x04, 0x05, 0x06]); - assert_eq!(&statement[87..119], &[0x33; 32]); - assert_eq!(&statement[119..121], &[0x07, 0x08]); - assert_eq!(&statement[121..123], &[0x09, 0x0A]); + #[test] + fn canonical_header_content_size_enforces_the_four_mib_boundary() { + let references = |count: usize, round: RoundNumber, domain: u8| { + (0..count) + .map(|index| { + let mut digest = [0; 32]; + digest[0] = domain; + digest[1..9].copy_from_slice(&(index as u64).to_be_bytes()); + BlockReference { + authority: index as AuthorityIndex % 4, + round, + digest: BlockDigest::from(digest), + } + }) + .collect::>() + }; + let maximum_total_references = (MAX_RBC_HEADER_CONTENT_SIZE + - RBC_HEADER_FIXED_CONTENT_SIZE) + / RBC_BLOCK_REFERENCE_SIZE; + let parent_count = maximum_total_references / 2; + let acknowledgment_count = maximum_total_references - parent_count; + let accepted = RbcCanonicalHeader::try_new( + 0, + 5, + references(parent_count, 4, 0x01), + references(acknowledgment_count, 3, 0x02), + 0, + TransactionsCommitment::default(), + ) + .unwrap(); + assert_eq!( + accepted + .encoded_content_size(accepted.acknowledgment_references().len()) + .unwrap(), + MAX_RBC_HEADER_CONTENT_SIZE - 32 + ); + + let mut too_large_acknowledgments = accepted.acknowledgment_references(); + too_large_acknowledgments.push(block(0, 3, 0x03)); + assert_eq!( + RbcCanonicalHeader::try_new( + accepted.reference.authority, + accepted.reference.round, + accepted.block_references.clone(), + too_large_acknowledgments, + accepted.meta_creation_time_ns, + accepted.transactions_commitment, + ), + Err(RbcError::HeaderContentTooLarge) + ); } #[test] @@ -1120,6 +3021,7 @@ mod tests { instance(TEST_INSTANCE_BYTE), BlockAuthenticationScheme::Ed25519, Arc::new(Vec::new()), + 0, ) .err() .unwrap(); @@ -1147,6 +3049,7 @@ mod tests { instance(TEST_INSTANCE_BYTE), BlockAuthenticationScheme::Ed25519, Arc::new(mac_keyrings_for_test(4)[0].clone()), + 0, ) .err() .unwrap(); @@ -1174,6 +3077,7 @@ mod tests { instance(TEST_INSTANCE_BYTE), BlockAuthenticationScheme::Ed25519, Arc::new(Vec::new()), + 0, ) .err() .unwrap(); @@ -1209,7 +3113,9 @@ mod tests { BlockAuthenticationScheme::MacVector, ); let block_ref = block(0, 7, 0x44); - let tag = author.make_initial_mac_tag(block_ref, 1).unwrap(); + let tag = author + .make_initial_mac_tag_for_reference(block_ref, 1) + .unwrap(); recipient .verify_initial_mac_tag(0, block_ref, &tag) @@ -1276,6 +3182,296 @@ mod tests { ); } + #[test] + fn typed_initial_proofs_gate_echo_for_every_authentication_scheme() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let canonical = valid_canonical_header(0, 5, 0x20); + let block_ref = canonical.reference(); + + for authentication in [ + BlockAuthenticationScheme::Ed25519, + BlockAuthenticationScheme::MlDsa44, + BlockAuthenticationScheme::MlDsa65, + BlockAuthenticationScheme::MacVector, + ] { + let mut receiver = kernel(Arc::clone(&committee), &keyrings, 1, authentication); + let proof = match authentication { + BlockAuthenticationScheme::Ed25519 => { + let digest = receiver.initial_signature_digest(block_ref).unwrap(); + RbcInitialProof::Ed25519(dummy_signer().sign_digest(&digest)) + } + BlockAuthenticationScheme::MlDsa44 => { + let digest = + BlockDigest::from(receiver.initial_signature_digest(block_ref).unwrap()); + RbcInitialProof::MlDsa44(dummy_ml_dsa_44_signer().sign_digest(&digest)) + } + BlockAuthenticationScheme::MlDsa65 => { + let digest = + BlockDigest::from(receiver.initial_signature_digest(block_ref).unwrap()); + RbcInitialProof::MlDsa65(dummy_ml_dsa_65_signer().sign_digest(&digest)) + } + BlockAuthenticationScheme::MacVector => { + let author = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::MacVector, + ); + RbcInitialProof::Mac( + author + .make_initial_mac_tag_for_reference(block_ref, 1) + .unwrap(), + ) + } + }; + let outcome = receiver + .accept_direct_initial_header(0, canonical.clone(), &proof) + .unwrap(); + assert!(matches!(outcome, + RbcInitialHeaderOutcome::Authenticated { effects } + if matches!(effects.as_slice(), [RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + .. + }]) + )); + } + } + + #[test] + fn invalid_initial_proof_still_allows_content_recovery_without_echo() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let canonical = valid_canonical_header(0, 5, 0x25); + let block_ref = canonical.reference(); + let mut receiver = kernel(committee, &keyrings, 1, BlockAuthenticationScheme::Ed25519); + let invalid = RbcInitialProof::Ed25519(SignatureBytes::default()); + + let outcome = receiver + .accept_direct_initial_header(0, canonical, &invalid) + .unwrap(); + assert!(matches!( + outcome, + RbcInitialHeaderOutcome::StagedUnauthenticated { + effects, + error: RbcError::InvalidInitialProof, + } if effects.is_empty() + )); + let slot = receiver.slot(&block_ref).unwrap(); + assert_eq!(slot.echoed, None); + assert_eq!( + slot.candidates + .get(&block_ref) + .and_then(|candidate| candidate.header.as_ref()) + .map(PinnedRbcHeader::reference), + Some(block_ref) + ); + + assert!( + receiver + .accept_recovered_header(valid_canonical_header(0, 5, 0x25)) + .unwrap() + .is_empty() + ); + + assert_eq!( + RbcInitialProof::from_block_authentication(&BlockAuthentication::MacVector(vec![])), + Err(RbcError::InvalidInitialProof) + ); + } + + #[test] + fn pinned_headers_and_echo_capabilities_are_kernel_bound() { + let committee_a = Committee::new_test(vec![1; 4]); + let committee_b = Committee::new_test(vec![1, 1, 1, 10]); + let keyrings = mac_keyrings_for_test(4); + let canonical = valid_canonical_header(0, 5, 0xA8); + let block_ref = canonical.reference(); + let pin_a = PinnedRbcHeader::validate(canonical.clone(), &committee_a).unwrap(); + assert_eq!( + PinnedRbcHeader::validate(canonical.clone(), &committee_b), + Err(RbcError::InvalidThresholdClock) + ); + + let mut committee_b_kernel = kernel( + committee_b, + &keyrings, + 1, + BlockAuthenticationScheme::Ed25519, + ); + assert_eq!( + committee_b_kernel.note_header_available(pin_a.clone()), + Err(RbcError::PinnedHeaderCommitteeMismatch) + ); + assert!(committee_b_kernel.slots.is_empty()); + + let author = kernel( + Arc::clone(&committee_a), + &keyrings, + 0, + BlockAuthenticationScheme::MacVector, + ); + let receiver_one = kernel( + Arc::clone(&committee_a), + &keyrings, + 1, + BlockAuthenticationScheme::MacVector, + ); + let proof = RbcInitialProof::Mac( + author + .make_initial_mac_tag_for_reference(block_ref, 1) + .unwrap(), + ); + let eligible_for_one = receiver_one + .direct_initial_header(0, pin_a.clone(), &proof) + .unwrap(); + let mut receiver_two = kernel( + Arc::clone(&committee_a), + &keyrings, + 2, + BlockAuthenticationScheme::MacVector, + ); + assert!(matches!( + receiver_two.accept_direct_initial_header(1, canonical, &proof), + Err(RbcError::InitialAuthorMismatch { .. }) + )); + assert!(receiver_two.slots.is_empty()); + assert_eq!( + receiver_two.accept_initial_header(eligible_for_one), + Err(RbcError::EchoCapabilityContextMismatch) + ); + assert!(receiver_two.slots.is_empty()); + + let eligible_for_instance = receiver_one + .direct_initial_header(0, pin_a, &proof) + .unwrap(); + let mut other_instance = kernel_with_instance( + committee_a, + &keyrings, + 1, + BlockAuthenticationScheme::MacVector, + 0xB6, + ); + assert_eq!( + other_instance.accept_initial_header(eligible_for_instance), + Err(RbcError::EchoCapabilityContextMismatch) + ); + assert!(other_instance.slots.is_empty()); + } + + #[test] + fn invalid_initial_proof_preserves_ready_effect_unblocked_by_staging() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let canonical = valid_canonical_header(0, 7, 0xB7); + let block_ref = canonical.reference(); + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 3, + BlockAuthenticationScheme::Ed25519, + ); + + for sender in 0..3 { + let message = phase_message( + Arc::clone(&committee), + &keyrings, + sender, + 3, + RbcPhase::Echo, + block_ref, + ); + receiver.handle_phase(sender, message).unwrap(); + } + let outcome = receiver + .accept_direct_initial_header( + 0, + canonical, + &RbcInitialProof::Ed25519(SignatureBytes::default()), + ) + .unwrap(); + assert!(matches!( + outcome, + RbcInitialHeaderOutcome::StagedUnauthenticated { + effects, + error: RbcError::InvalidInitialProof, + } if effects == vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + }] + )); + let slot = receiver.slot(&block_ref).unwrap(); + assert_eq!(slot.echoed, None); + assert_eq!(slot.readied, Some(block_ref)); + assert!(slot.candidates[&block_ref].header.is_some()); + } + + #[test] + fn local_initial_constructor_binds_the_local_author_and_content() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut kernel = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + ); + let template = valid_canonical_header(1, 5, 0x27); + + let local = kernel + .start_local_initial_header( + template.reference.round, + template.block_references.clone(), + template.acknowledgment_references(), + template.meta_creation_time_ns, + template.transactions_commitment, + ) + .unwrap(); + assert!(kernel.make_local_initial_signature_digest(&local).is_ok()); + let other_instance = kernel_with_instance( + committee, + &keyrings, + 0, + BlockAuthenticationScheme::Ed25519, + 0xB8, + ); + assert_eq!( + other_instance.make_local_initial_signature_digest(&local), + Err(RbcError::LocalInitialContextMismatch) + ); + assert!(matches!( + kernel.start_local_initial_header( + template.reference.round, + template.block_references.clone(), + template.acknowledgment_references(), + template.meta_creation_time_ns + 1, + template.transactions_commitment, + ), + Err(RbcError::ConflictingInitialHeader { .. }) + )); + let (pinned, effects) = local.into_parts(); + assert_eq!(pinned.reference().authority, 0); + assert_ne!(pinned.reference(), template.reference()); + assert!(matches!( + effects.as_slice(), + [RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + .. + }] + )); + + assert!(matches!( + kernel.start_local_initial_header( + 6, + template.block_references[..2].to_vec(), + Vec::new(), + template.meta_creation_time_ns, + template.transactions_commitment, + ), + Err(RbcError::InvalidThresholdClock) + )); + } + #[test] fn phase_messages_are_specialized_for_each_recipient() { let committee = Committee::new_test(vec![1; 4]); @@ -1339,7 +3535,9 @@ mod tests { sender.make_phase_message(RbcPhase::Echo, first, 1), Err(RbcError::PhaseNotAuthorized { .. }) )); - sender.note_header_available(first).unwrap(); + sender + .note_header_available(pinned_header(&committee, first)) + .unwrap(); assert!(matches!( sender.make_phase_message(RbcPhase::Echo, first, 1), Err(RbcError::PhaseNotAuthorized { .. }) @@ -1358,7 +3556,9 @@ mod tests { sender.make_phase_message(RbcPhase::Echo, first, 1).unwrap() ); - sender.note_header_available(conflicting).unwrap(); + sender + .note_header_available(pinned_header(&committee, conflicting)) + .unwrap(); assert!(sender.authorize_echo(conflicting).unwrap().is_empty()); assert!(matches!( sender.make_phase_message(RbcPhase::Echo, conflicting, 1), @@ -1542,7 +3742,9 @@ mod tests { let mut message = sender .make_phase_message(RbcPhase::Echo, block_ref, 1) .unwrap(); - message.tag = sender.make_initial_mac_tag(block_ref, 1).unwrap(); + message.tag = sender + .make_initial_mac_tag_for_reference(block_ref, 1) + .unwrap(); assert_eq!( receiver.handle_phase(0, message), @@ -1642,7 +3844,9 @@ mod tests { ); assert_eq!( - receiver.note_header_available(block_ref).unwrap(), + receiver + .note_header_available(pinned_header(&committee, block_ref)) + .unwrap(), vec![RbcEffect::MulticastPhase { phase: RbcPhase::Ready, block_ref, @@ -1685,13 +3889,15 @@ mod tests { )); assert_eq!( - receiver.note_header_available(block_ref).unwrap(), + receiver + .note_header_available(pinned_header(&committee, block_ref)) + .unwrap(), vec![ RbcEffect::MulticastPhase { phase: RbcPhase::Ready, block_ref, }, - RbcEffect::Deliver(block_ref), + RbcEffect::Deliver(pinned_header(&committee, block_ref)), ] ); } @@ -1731,7 +3937,14 @@ mod tests { initial_request.first().cloned() ); - let third = phase_message(committee, &keyrings, 2, 3, RbcPhase::Ready, block_ref); + let third = phase_message( + Arc::clone(&committee), + &keyrings, + 2, + 3, + RbcPhase::Ready, + block_ref, + ); let expanded_request = receiver.handle_phase(2, third).unwrap(); assert!(matches!( expanded_request.as_slice(), @@ -1742,7 +3955,9 @@ mod tests { expanded_request.first().cloned() ); - receiver.note_header_available(block_ref).unwrap(); + receiver + .note_header_available(pinned_header(&committee, block_ref)) + .unwrap(); assert_eq!(receiver.retry_header_request(block_ref).unwrap(), None); } @@ -1772,13 +3987,15 @@ mod tests { assert_eq!(receiver.slot(&block_ref).unwrap().delivered, None); assert_eq!( - receiver.note_header_available(block_ref).unwrap(), + receiver + .note_header_available(pinned_header(&committee, block_ref)) + .unwrap(), vec![ RbcEffect::MulticastPhase { phase: RbcPhase::Ready, block_ref, }, - RbcEffect::Deliver(block_ref), + RbcEffect::Deliver(pinned_header(&committee, block_ref)), ] ); } @@ -1794,7 +4011,9 @@ mod tests { 3, BlockAuthenticationScheme::Ed25519, ); - receiver.note_header_available(block_ref).unwrap(); + receiver + .note_header_available(pinned_header(&committee, block_ref)) + .unwrap(); for sender in [0, 0, 1] { let message = phase_message( @@ -1836,7 +4055,9 @@ mod tests { 0, BlockAuthenticationScheme::Ed25519, ); - receiver.note_header_available(first).unwrap(); + receiver + .note_header_available(pinned_header(&committee, first)) + .unwrap(); assert!(matches!( receiver.authorize_echo(first).unwrap().as_slice(), [RbcEffect::MulticastPhase { @@ -1844,7 +4065,9 @@ mod tests { .. }] )); - receiver.note_header_available(quorum_candidate).unwrap(); + receiver + .note_header_available(pinned_header(&committee, quorum_candidate)) + .unwrap(); assert!( receiver .authorize_echo(quorum_candidate) @@ -1877,7 +4100,7 @@ mod tests { } #[test] - fn evidence_is_per_candidate_but_delivery_is_slot_global() { + fn phase_equivocation_is_ignored_and_delivery_is_slot_global() { let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(4); let first = block(0, 10, 0x81); @@ -1888,8 +4111,12 @@ mod tests { 3, BlockAuthenticationScheme::Ed25519, ); - receiver.note_header_available(first).unwrap(); - receiver.note_header_available(second).unwrap(); + receiver + .note_header_available(pinned_header(&committee, first)) + .unwrap(); + receiver + .note_header_available(pinned_header(&committee, second)) + .unwrap(); for candidate in [first, second] { let message = phase_message( @@ -1904,7 +4131,7 @@ mod tests { } assert!(receiver.candidate(&first).unwrap().echoes.votes.contains(0)); assert!( - receiver + !receiver .candidate(&second) .unwrap() .echoes @@ -1943,8 +4170,10 @@ mod tests { fn four_kernel_split_initial_values_converge_on_at_most_one_delivery() { let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(4); - let first = block(0, 12, 0xA1); - let conflicting = block(0, 12, 0xA2); + let first_header = valid_canonical_header(0, 12, 0xA1); + let conflicting_header = valid_canonical_header(0, 12, 0xA2); + let first = first_header.reference(); + let conflicting = conflicting_header.reference(); let mut kernels: Vec<_> = (0..4) .map(|authority| { kernel( @@ -1955,19 +4184,53 @@ mod tests { ) }) .collect(); + let mut header_stores = vec![AHashMap::new(); 4]; let mut initial_effects = Vec::new(); - for (authority, block_ref) in [(0, first), (1, first), (2, first), (3, conflicting)] { - kernels[authority as usize] - .note_header_available(block_ref) - .unwrap(); - let effects = kernels[authority as usize] - .authorize_echo(block_ref) + let local = kernels[0] + .start_local_initial_header( + first_header.reference.round, + first_header.block_references.clone(), + first_header.acknowledgment_references(), + first_header.meta_creation_time_ns, + first_header.transactions_commitment, + ) + .unwrap(); + assert_eq!(local.header(), &first_header); + let first_signature_digest = kernels[0] + .make_local_initial_signature_digest(&local) + .unwrap(); + let first_proof = + RbcInitialProof::Ed25519(dummy_signer().sign_digest(&first_signature_digest)); + let (local_header, local_effects) = local.into_parts(); + header_stores[0].insert(first, local_header.header().clone()); + initial_effects.push((0, local_effects)); + + for authority in 1..3 { + let outcome = kernels[authority as usize] + .accept_direct_initial_header(0, first_header.clone(), &first_proof) .unwrap(); + let RbcInitialHeaderOutcome::Authenticated { effects } = outcome else { + panic!("valid direct initial header must authenticate") + }; + header_stores[authority as usize].insert(first, first_header.clone()); initial_effects.push((authority, effects)); } - let deliveries = pump_phase_effects(&mut kernels, initial_effects, true); + let conflicting_digest = kernels[3].initial_signature_digest(conflicting).unwrap(); + let conflicting_proof = + RbcInitialProof::Ed25519(dummy_signer().sign_digest(&conflicting_digest)); + let outcome = kernels[3] + .accept_direct_initial_header(0, conflicting_header.clone(), &conflicting_proof) + .unwrap(); + let RbcInitialHeaderOutcome::Authenticated { effects } = outcome else { + panic!("valid conflicting author proof must authenticate at its recipient") + }; + header_stores[3].insert(conflicting, conflicting_header); + initial_effects.push((3, effects)); + + let (deliveries, recoveries) = + pump_phase_effects(&mut kernels, initial_effects, &mut header_stores); assert!(deliveries.iter().all(|delivered| delivered == &[first])); assert!( deliveries @@ -1975,6 +4238,11 @@ mod tests { .flatten() .all(|delivered| *delivered != conflicting) ); + assert!( + recoveries + .iter() + .any(|(requester, _, block_ref)| *requester == 3 && *block_ref == first) + ); assert_eq!(kernels[3].slot(&first).unwrap().echoed, Some(conflicting)); assert_eq!(kernels[3].slot(&first).unwrap().readied, Some(first)); } @@ -1983,7 +4251,8 @@ mod tests { fn poisoned_initial_mac_does_not_block_rbc_totality() { let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(4); - let block_ref = block(0, 13, 0xA3); + let canonical = valid_canonical_header(0, 13, 0xA3); + let block_ref = canonical.reference(); let mut kernels: Vec<_> = (0..4) .map(|authority| { kernel( @@ -1994,33 +4263,53 @@ mod tests { ) }) .collect(); - - let valid_for_one = kernels[0].make_initial_mac_tag(block_ref, 1).unwrap(); - let valid_for_two = kernels[0].make_initial_mac_tag(block_ref, 2).unwrap(); - kernels[1] - .verify_initial_mac_tag(0, block_ref, &valid_for_one) - .unwrap(); - kernels[2] - .verify_initial_mac_tag(0, block_ref, &valid_for_two) + let mut header_stores = vec![AHashMap::new(); 4]; + + let local = kernels[0] + .start_local_initial_header( + canonical.reference.round, + canonical.block_references.clone(), + canonical.acknowledgment_references(), + canonical.meta_creation_time_ns, + canonical.transactions_commitment, + ) .unwrap(); - assert_eq!( - kernels[3].verify_initial_mac_tag(0, block_ref, &valid_for_two), - Err(RbcError::InvalidInitialTag) - ); - - let mut initial_effects = Vec::new(); - for authority in 0..3 { - kernels[authority as usize] - .note_header_available(block_ref) - .unwrap(); - let effects = kernels[authority as usize] - .authorize_echo(block_ref) + let valid_for_one = kernels[0].make_local_initial_mac_tag(&local, 1).unwrap(); + let valid_for_two = kernels[0].make_local_initial_mac_tag(&local, 2).unwrap(); + let (local_header, local_effects) = local.into_parts(); + header_stores[0].insert(block_ref, local_header.header().clone()); + let mut initial_effects = vec![(0, local_effects)]; + for (authority, tag) in [(1, valid_for_one), (2, valid_for_two)] { + let outcome = kernels[authority as usize] + .accept_direct_initial_header(0, canonical.clone(), &RbcInitialProof::Mac(tag)) .unwrap(); + let RbcInitialHeaderOutcome::Authenticated { effects } = outcome else { + panic!("recipient-specific author MAC must authenticate") + }; + header_stores[authority as usize].insert(block_ref, canonical.clone()); initial_effects.push((authority, effects)); } - let deliveries = pump_phase_effects(&mut kernels, initial_effects, true); + let poisoned = kernels[3] + .accept_direct_initial_header( + 0, + canonical.clone(), + &RbcInitialProof::Mac(valid_for_two), + ) + .unwrap(); + assert!(matches!( + poisoned, + RbcInitialHeaderOutcome::StagedUnauthenticated { + effects, + error: RbcError::InvalidInitialTag, + } if effects.is_empty() + )); + header_stores[3].insert(block_ref, canonical); + + let (deliveries, recoveries) = + pump_phase_effects(&mut kernels, initial_effects, &mut header_stores); assert!(deliveries.iter().all(|delivered| delivered == &[block_ref])); + assert!(recoveries.is_empty()); let recovered_slot = kernels[3].slot(&block_ref).unwrap(); assert_eq!(recovered_slot.echoed, None); assert_eq!(recovered_slot.readied, Some(block_ref)); @@ -2040,7 +4329,9 @@ mod tests { 3, BlockAuthenticationScheme::Ed25519, ); - receiver.note_header_available(block_ref).unwrap(); + receiver + .note_header_available(pinned_header(&committee, block_ref)) + .unwrap(); assert_eq!( receiver.authorize_echo(block_ref).unwrap(), vec![RbcEffect::MulticastPhase { diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index df6a36ec..c988c668 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -597,7 +597,7 @@ impl BlockHeader { } } -fn expand_acknowledgments( +pub(crate) fn expand_acknowledgments( block_references: &[BlockReference], acknowledgment_intersection: Option, acknowledgment_references: &[BlockReference], @@ -625,7 +625,7 @@ fn count_acknowledgments( (block_references.len() - start) + acknowledgment_references.len() } -fn compress_acknowledgments( +pub(crate) fn compress_acknowledgments( block_references: &[BlockReference], acknowledgment_references: &[BlockReference], ) -> (Option, Vec) { diff --git a/docs/starfish-rbc-protocol.md b/docs/starfish-rbc-protocol.md index 216102b7..e7eb4d8f 100644 --- a/docs/starfish-rbc-protocol.md +++ b/docs/starfish-rbc-protocol.md @@ -1,6 +1,7 @@ # Starfish-RBC protocol specification -Status: protocol specification with isolated RBC kernel; network and DAG integration pending +Status: protocol specification with isolated RBC kernel and canonical header staging; network and +DAG integration pending This document specifies the first correctness-oriented prototype of Starfish with reliable header certification and a signature-free MAC configuration. The provisional CLI name is `starfish-rbc`. @@ -102,16 +103,40 @@ The value proposed in a slot is a canonical Starfish header. Its identifier rema BlockReference = (author, round, content_digest) ``` -`content_digest` must commit unambiguously to the canonical header content, including the -transaction commitment, parent references, acknowledgments, and other consensus-relevant fields. -It does not commit to the initial authentication sidecar or to reliable-broadcast messages. +`content_digest` commits unambiguously to the canonical header content, including the transaction +commitment, parent references, and logical acknowledgments. It does not commit to the initial +authentication sidecar, protocol instance, committee ID, reliable-broadcast messages, transaction +payload, or the transport representation of compressed acknowledgments. -The current shared `BlockDigest` implementation hashes the parent-reference vector immediately +The legacy shared `BlockDigest` implementation hashes the parent-reference vector immediately followed by the acknowledgment vector without encoding their lengths. Moving a reference across -that boundary can therefore preserve the digest without finding a BLAKE3 collision. Before -Starfish-RBC accepts headers, milestone three must add a Starfish-RBC canonical content encoding -with explicit field markers and collection lengths. This is a blocking proof obligation, not an -optional optimization. +that boundary can therefore preserve the digest without finding a BLAKE3 collision. Starfish-RBC +does not change that legacy encoding. It uses plain BLAKE3 over this canonical tagged byte stream: + +```text +0x01 || author:u16_be +0x02 || round:u32_be +0x03 || parent_count:u32_be || parent[0] || ... || parent[n-1] +0x04 || logical_ack_count:u32_be || ack[0] || ... || ack[m-1] +0x05 || creation_time_ns:u64_be +0x06 || transactions_commitment:[u8;32] + +reference = authority:u16_be || round:u32_be || digest:[u8;32] +``` + +There is deliberately no `starfish:block-ref:v2` string, protocol name, session identifier, or +authentication mode in this input: a block reference remains exactly `H(content)`. The field tags +and fixed-width counts make the content grammar unambiguous. Any future content field requires a +new canonical encoding rather than silently extending this one. + +Acknowledgments remain compressed on the wire so that RBC does not inflate the Starfish bandwidth +baseline. Before hashing, they are expanded into their logical ordered vector. The stored +compression must be canonical: the intersection is in range, denotes the maximal shared parent +suffix, and recompressing the logical vector must reproduce the stored fields byte-for-byte. An +intersection up to 255 uses the compact form; a larger index uses the legacy full-vector fallback. +Local construction follows existing Starfish semantics: a shared parent suffix is normalized to +the front of the logical vector and non-shared acknowledgments retain their relative order. Exact +duplicate parents and acknowledgments are rejected before this normalization. `protocol_instance` and `committee_id` are authenticated domain-separation inputs but are not added to `BlockReference`: @@ -132,13 +157,16 @@ effective validity/quorum thresholds or information length disagree with the can Header processing has three distinct gates: -1. **Content validation** is deterministic and view-independent. It checks the canonical digest, - intrinsic field relationships, committee/round bounds, the committed transaction root, and - protocol syntax without requiring the local initial proof, transaction data, a shard, or locally - available dependencies. -2. **Initial authentication** determines only whether this validator may send ECHO for the +1. **Structural content validation** is deterministic and view-independent. It checks the canonical + digest, intrinsic field relationships, committee membership, the previous-round quorum clock, + the committed transaction root, and protocol syntax without requiring the local initial proof, + transaction data, a shard, or locally available dependencies. The resulting pin caches the + already-validated committee ID; it does not rehash the committee for every header. +2. **Ingress admission** applies the local future-round ceiling and the monotonic new-slot floor + before candidate allocation. These local resource checks are not part of content identity. +3. **Initial authentication** determines only whether this validator may send ECHO for the candidate. -3. **DAG admission and clean activation** resolve dependencies and determine whether the delivered +4. **DAG admission and clean activation** resolve dependencies and determine whether the delivered header can influence Starfish. The implementation must not keep these gates bundled in the current all-or-nothing block verifier. @@ -173,6 +201,12 @@ The local author does not need to authenticate its header to itself. Local const author's ECHO eligibility evidence; only messages sent to other validators need receiver-specific initial tags. +The atomic local-start operation returns an unforgeable handle only after the selected header is +pinned and local ECHO is recorded. Author-side signature-digest and MAC-tag generation require a +borrow of that handle and recheck the selected stored reference. The service generates all INIT +proofs while borrowing the handle, then consumes it to obtain the pinned header and queued phase +effects; arbitrary caller-supplied references cannot reach the author-side proof API. + A valid initial proof permits an honest recipient to ECHO the header. It does not by itself make the header clean, globally available, or safe to commit. @@ -208,10 +242,13 @@ A forwarded or replayed phase message received from another peer never counts, e contain a valid tag addressed to the receiver. Local ECHO/READY actions are counted locally and do not require a loopback network message. -The isolated kernel enforces committee membership and rejects genesis slots. The active/retained -round-window check requires the service's current round and is therefore an explicit ingress -adapter responsibility in milestone three; it must run before calling the kernel or allocating a -candidate. +The kernel enforces committee membership, rejects genesis slots, and rejects a header or phase +reference more than 100 rounds ahead of its monotonic local round before allocating candidate +state. It also exposes a separate monotonic floor for allocating previously unseen slots. That +floor defaults to round one and must not be derived automatically from DAG progress: the +integration layer may advance it only from a proved safe recovery/retirement watermark. Existing +slots remain active below the floor so late evidence for a newly observed candidate can still +complete reliable-delivery totality. The kernel rejects a floor above its current local round. ## 4. Messages @@ -277,8 +314,7 @@ SlotState { } CandidateState { - header: Option, - initial_authentication_valid: bool, + header: Option, echo_senders: AuthoritySet, ready_senders: AuthoritySet, echo_quorum_observed: bool, @@ -290,12 +326,21 @@ CandidateState { } ``` -The isolated milestone-two kernel temporarily represents header presence with an internal boolean -to test transitions. Its header-available and initial-proof authorization hooks are module-private, -so another crate module cannot assert these facts by call ordering. Milestone three must replace the -boolean with a typed handle that owns or pins the canonical content-validated header, and expose -only combined typed ingress paths. Cache eviction must not leave the kernel advertising a header it -no longer retains. +The kernel stores an `Arc`-backed `PinnedRbcHeader`, not a presence boolean. Its constructor checks +the canonical digest, acknowledgment representation, intrinsic rounds, committee membership, +previous-round quorum clock, required transaction commitment, forbidden protocol extensions, and +resource bounds. Every pin records the committee ID under which those checks ran; another committee +cannot consume it. The pin is held for as long as this validator may advertise itself as a header +holder, so cache eviction cannot silently invalidate the RBC availability invariant. + +A distinct `EchoEligibleHeader` capability is bound to the full RBC context and local recipient. It +is created only by valid direct-author initial authentication or the local-author construction path. +Recovered content can be pinned and can unblock READY/delivery, but it cannot be converted into +ECHO eligibility. Direct INIT ingress stages content even when the receiver-specific proof is +invalid and returns all effects unblocked by staging; this makes poisoned-tag handling atomic rather +than an adapter convention. A proposal received from a peer other than its claimed author is +rejected by this direct-INIT path. RBC delivery carries the pinned header rather than only its +reference. The `echoed`, `readied`, and `delivered` guards are slot-global, not per digest. Evidence is kept per digest so that Byzantine equivocation can be observed without locking the receiver to the first @@ -312,15 +357,29 @@ In particular: For correctness in version one, slot-global locks, phase evidence, threshold latches, and headers advertised by an honest local phase action are retained for the whole benchmark run. Unsupported candidate bodies may be evicted because they can be fetched again by digest, but eviction must not -discard evidence or a pending transition. A proved retirement boundary and adversarial storage -bounds are deferred with crash/restart support; arbitrary cache eviction is not a protocol action. +discard evidence or a pending transition. The kernel's new-slot floor is not advanced in the first +benchmark prototype. A proved nontrivial retirement boundary is deferred with crash/restart +support; arbitrary cache eviction is not a protocol action. + +Each direct sender may sponsor only its first ECHO value and first READY value in a slot. Exact +retransmissions are idempotent; later phase equivocations from that sender are verified but ignored +before candidate allocation. ECHO and READY use separate ledgers because an honest validator may +ECHO one value and later READY another. Together with one content-only direct-author candidate, +this bounds retained candidates per slot to at most `2 * committee_size - 1`: one initial candidate +plus two first-phase values from each of the other validators. A recovered header can attach only to +an already retained candidate and cannot allocate one by itself. + +Version one accepts at most 65,535 parents or logical acknowledgments per field and at most 4 MiB +of canonical header content. Bounded vector deserialization enforces the serialized parent and +extra-acknowledgment limits incrementally, before an attacker can force an unbounded vector +allocation. Sending a local phase is one atomic state transition: set the slot-global guard, insert the local authority into that candidate's phase-sender set, and enqueue separately authenticated messages for all other validators. Threshold checks include this local evidence. Message materialization checks the recorded slot-global guard and the kernel's header-present predicate again; it cannot -authenticate a phase for an unauthorized or conflicting value. Milestone three makes that predicate -a pinned-header handle. The same authorized phase may be materialized again for retransmission. +authenticate a phase for an unauthorized or conflicting value. That predicate is a pinned-header +handle. The same authorized phase may be materialized again for retransmission. ## 6. State machine @@ -330,13 +389,17 @@ On `HeaderProposal` for candidate `R`: 1. Validate committee membership, slot consistency, header syntax, canonical content digest, protocol-specific fields, and resource bounds. -2. Store the fixed-size-checked, content-valid header as a candidate even if its local initial - authenticator is missing or invalid. This permits later RBC delivery to repair a poisoned - recipient tag. -3. Verify that version one's proposal was received directly from the claimed author, then verify the - selected initial authentication method for the local recipient. For the local author's own - header, successful local construction supplies this evidence without a loopback signature or - MAC. +2. Verify that version one's direct-INIT peer is the claimed author. A header from another peer does + not enter the content-only INIT allowance; it may be accepted later only as recovery for an + already phase-evidenced candidate. +3. Store the fixed-size-checked, content-valid direct-author header as a candidate even if its local + initial authenticator is missing or invalid. Return any READY/delivery effects unblocked by that + staging together with the authentication failure. This permits later RBC delivery to repair a + poisoned recipient tag without relying on adapter call ordering. For the local author's own + header, one mutable kernel operation fixes the author and digest, selects and pins the candidate, + records local ECHO, and only then exposes a context-bound handle for proof generation and + dissemination. This prevents two conflicting local proposals from escaping before the slot lock + is installed. 4. If the direct-author check and proof are valid and the slot-global ECHO guard is empty, record the local ECHO immediately and send a recipient-specific ECHO for `R` to every other validator. Header RBC does not wait for parents, acknowledgments, transaction data, or a shard to arrive. @@ -344,16 +407,24 @@ On `HeaderProposal` for candidate `R`: its dirty dependencies are present. This follows Sailfish's dirty/clean setup and does not make the candidate consensus-visible. -A relayed header may be retained as a content-valid candidate, but it does not trigger ECHO in -version one. Tree dissemination later changes this eligibility rule to accept a relayed -receiver-specific author proof and must extend the integrity argument accordingly. +A relayed header may be retained through the recovery path for an already phase-evidenced +candidate, but it does not trigger ECHO in version one. Tree dissemination later changes this +eligibility rule to accept a relayed receiver-specific author proof and must extend the integrity +argument accordingly. + +The first content-valid INIT received directly from the claimed author occupies the one +content-only INIT allowance even if its proof is invalid. A later proof for the same reference may +still authenticate and ECHO it; a conflicting author proposal cannot consume more memory or replace +it. This is deliberate: a Byzantine author is not promised validity for its slot, while a different +value can still complete through first-phase evidence and header recovery. ### 6.2 Receiving ECHO On a valid direct `Echo(R, S)`: -1. Record `S` once in `R.echo_senders`. A Byzantine sender may appear in the evidence sets of - multiple conflicting candidates, but its stake counts only once per candidate. +1. If this is `S`'s first ECHO value in the slot, record `S` once in `R.echo_senders`. An exact + retransmission is idempotent. A later conflicting ECHO from `S` is ignored before allocating or + changing candidate evidence. 2. When ECHO stake for `R` reaches `Q`, latch `echo_quorum_observed` and: - if the header is absent, request it from multiple recorded ECHO senders; - validate and store the returned header; and @@ -371,7 +442,8 @@ honest-author progress by withholding theirs. On a valid direct `Ready(R, S)`: -1. Record `S` once in `R.ready_senders`. +1. If this is `S`'s first READY value in the slot, record `S` once in `R.ready_senders`. Treat an + exact retransmission as idempotent and ignore a later conflicting READY before allocation. 2. When READY stake for `R` reaches `V`, latch `ready_validity_observed` and: - if the header is absent, request it from multiple recorded READY senders; - validate and store the returned header; and @@ -634,6 +706,13 @@ Native Starfish with Ed25519 and ML-DSA should also be measured separately. That the total cost of reliable delivery, but it must not be presented as an isolated authentication comparison because the message flows differ. +The existing unsafe `starfish-mac` lower bound sends the full committee-sized MAC vector with each +direct-author streamed header, while Starfish-RBC/MAC sends only the recipient's tag and adds +ECHO/READY traffic. Their measured delta is therefore the net whole-protocol cost, not a pure RBC +overhead number: saved INIT bytes can hide part of the phase-message cost. The benchmark must report +INIT/header bytes and ECHO/READY bytes separately; a one-tag lower-bound projection may be added but +must be labeled as such. + Metrics should separate: - initial header-authentication bytes and CPU; @@ -679,11 +758,14 @@ Each milestone is committed separately. committee identity, slot-global ECHO/READY state, guarded recipient-specific message materialization, retryable header-holder tracking, and local plus four-kernel adversarial tests. The synchronous kernel is intentionally not network- or DAG-wired yet. -3. **Header staging and retrieval:** split content validation from initial authentication, add - a length-delimited Starfish-RBC content digest, typed/pinned fixed-size-checked candidate staging, - retained-window filtering, pending triggers, and durable fetching from direct phase senders. -4. **Certified Starfish integration:** add `starfish-rbc`, selectable initial authentication, - dirty/clean lifecycle, clean-only acknowledgments, and clean-only consensus/linearization. +3. **Canonical header boundary (complete):** split content validation from initial authentication, + add the tagged length-delimited content digest, canonical compressed acknowledgments, + committee-bound pinned headers, context-bound ECHO capabilities, atomic poisoned-proof staging, + bounded phase equivocation, round admission seams, pending triggers, and holder-backed + multi-kernel recovery tests. The durable network fetch owner is part of milestone four. +4. **Certified Starfish integration:** add `starfish-rbc`, selectable initial authentication, the + network RBC service and durable multi-holder fetch retry, dirty/clean lifecycle, clean-only + acknowledgments, and clean-only consensus/linearization. 5. **End-to-end validation:** poisoned-tag, equivocation, dangling-parent, and all-authentication commit tests. 6. **Tree dissemination:** subtree tag bundles, redundant routing/fallback, and matching signature @@ -694,10 +776,10 @@ Each milestone is committed separately. ## 15. Remaining integration decisions -The kernel behavior and authenticated encoding above are fixed. Integration must still choose: +The kernel behavior, content digest, authenticated encoding, size limits, and admission semantics +above are fixed. Integration must still choose: - how benchmark genesis generates and distributes the fresh 32-byte `protocol_instance`; -- the exact field markers and collection-length widths for the Starfish-RBC content digest; - a safe post-v1 state-retirement and garbage-collection rule; - header-holder request fanout and retry timing; - whether legacy unsafe `*-mac` aliases are renamed or retained as lower-bound benchmarks. From 5214c7b38dd1dbd3b2f7c415728a8b50816f355d Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:05:44 +0200 Subject: [PATCH 24/62] Integrate Starfish-RBC protocol --- README.md | 20 +- crates/orchestrator/src/main.rs | 21 +- crates/orchestrator/src/protocol/starfish.rs | 39 +- crates/starfish-core/src/block_manager.rs | 148 +- crates/starfish-core/src/broadcaster.rs | 30 +- crates/starfish-core/src/config.rs | 46 + .../src/consensus/base_committer.rs | 121 +- .../starfish-core/src/consensus/linearizer.rs | 97 +- .../src/consensus/universal_committer.rs | 6 + crates/starfish-core/src/core.rs | 218 ++- .../starfish-core/src/core_thread/spawned.rs | 29 +- crates/starfish-core/src/dag_state.rs | 482 ++++++- crates/starfish-core/src/lib.rs | 3 +- crates/starfish-core/src/metrics.rs | 16 + crates/starfish-core/src/net_sync.rs | 238 +++- crates/starfish-core/src/network.rs | 100 +- crates/starfish-core/src/starfish_rbc.rs | 226 ++- .../starfish-core/src/starfish_rbc_service.rs | 1225 +++++++++++++++++ crates/starfish-core/src/syncer.rs | 49 +- crates/starfish-core/src/types.rs | 157 ++- crates/starfish-core/src/validator.rs | 89 +- crates/starfish/src/main.rs | 40 +- docs/starfish-rbc-protocol.md | 46 +- 23 files changed, 3255 insertions(+), 191 deletions(-) create mode 100644 crates/starfish-core/src/starfish_rbc_service.rs diff --git a/README.md b/README.md index 5b2db961..85d19d07 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository is a benchmarking framework for DAG-based BFT consensus protocols in the partially synchronous model, implemented in Rust. -It includes 9 protocol implementations with configurable +It includes 10 protocol implementations with configurable dissemination strategies, storage backends, and Byzantine fault injection. @@ -22,6 +22,7 @@ injection. | Starfish-Speed | `starfish-speed` | 4.5δ | Uncertified | Encoded | Push | O(n⁴) | O(n⁴) | -- | | Sparse-Starfish-Speed | `sparse-starfish-speed` | 4.5δ | Uncertified | Encoded | Push | O(n²) | O(n³) | -- | | Starfish | `starfish` | 5.5δ | Uncertified | Encoded | Push | O(n⁴) | O(n⁴) | [eprint.iacr.org/2025/567](https://eprint.iacr.org/2025/567) | +| Starfish-RBC (prototype) | `starfish-rbc` | TBD | RBC-certified headers | Encoded | Push | TBD | TBD | [design](docs/starfish-rbc-protocol.md) | | Cordial Miners | `cordial-miners` | 6δ | Uncertified | Full | Push | O(n³) | O(n⁴) | [arxiv.org/pdf/2205.09174](https://arxiv.org/pdf/2205.09174) | | Sailfish++ | `sailfish-pp` | 6δ | Certified | Full | Pull | O(n³) | O(n⁴) | [arxiv.org/abs/2505.02761](https://arxiv.org/abs/2505.02761) | | Starfish-BLS | `starfish-bls` | 6.5δ | Uncertified | Encoded | Push | O(n²) | O(n³) | [eprint.iacr.org/2025/567](https://eprint.iacr.org/2025/567)* | @@ -43,6 +44,10 @@ and compressed block references. certificate tracking, similar in architecture to Mysticeti-BLS, but with cheaper certification. **Starfish** uses push dissemination for headers and Reed-Solomon encoded shards with acknowledgment references between validators. +**Starfish-RBC** composes plain Starfish with direct Bracha reliable broadcast of canonical +headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use +Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research +prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's @@ -61,13 +66,15 @@ offloaded from the critical path. ### Block authentication -Every consensus protocol can select its block signature independently: +Every consensus protocol can select its public block-signature scheme independently. +Starfish-RBC additionally supports a receiver-specific MAC for the author's initial header: | Scheme | CLI option | |---|---| | Ed25519 (default) | `--block-authentication ed25519` or omit the option | | ML-DSA-44 | `--block-authentication ml-dsa-44` | | ML-DSA-65 | `--block-authentication ml-dsa-65` | +| Pairwise MAC (Starfish-RBC only) | `--block-authentication mac` | For example, `--consensus mysticeti --block-authentication ml-dsa-65` changes Mysticeti's block signature without creating another consensus protocol. This @@ -93,6 +100,11 @@ treated as production-ready cryptography. #### Experimental MAC protocols +`--consensus starfish-rbc --block-authentication mac` runs the reliable-header-broadcast +prototype. It sends one author tag to each intended recipient, then runs the same pairwise-MAC +ECHO/READY flow used by Starfish-RBC's signature-authenticated modes. Only locally delivered, +dependency-closed headers enter the clean consensus DAG. + `starfish-mac`, `starfish-speed-mac`, `sparse-starfish-speed-mac`, and `bluestreak-mac` remain separate work-in-progress benchmark protocols. They are not interchangeable signature selections and cannot be combined with @@ -102,8 +114,8 @@ These variants measure a lower bound for pairwise-MAC authentication. Direct author streaming carries the full committee-sized MAC vector; relays and synchronization responses carry only the destination's tag. Pairwise MACs do not provide transferable authorship, and a Byzantine author can give different -recipients valid and invalid tags for the same block reference. The current -prototype does not add the quorum-authentication/RBC exchange needed to bind +recipients valid and invalid tags for the same block reference. These lower-bound modes do not add +the quorum-authentication/RBC exchange needed to bind the author to an available authenticator. It therefore makes no safety or liveness claim and must not be treated as a proven variant of the underlying protocol. diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index ebd04de2..1e39dcc9 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -60,7 +60,7 @@ pub struct Opts { /// Block signature scheme used by every selected consensus protocol. /// Defaults to Ed25519. Not applicable to experimental `*-mac` protocols. - #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65", global = true)] + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, /// The type of operation to run. @@ -2323,6 +2323,25 @@ mod tests { } } + #[test] + fn benchmark_parses_starfish_rbc_mac_authentication() { + let opts = Opts::try_parse_from([ + "orchestrator", + "benchmark", + "--block-authentication", + "mac", + "--protocols", + "starfish-rbc", + ]) + .unwrap(); + + assert_eq!(opts.block_authentication.as_deref(), Some("mac")); + let Operation::Benchmark { protocols, .. } = opts.operation else { + panic!("expected benchmark operation"); + }; + assert_eq!(protocols, vec!["starfish-rbc"]); + } + #[test] fn committee_sweep_parses_grouped_protocols_and_sizes() { let opts = Opts::try_parse_from([ diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 7d0b0e86..11a7fe72 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -116,7 +116,10 @@ impl ProtocolCommands for StarfishProtocol { .collect::>() .join(" "); - let node_parameters = parameters.node_parameters.clone(); + let node_parameters = Self::node_parameters_for_genesis( + ¶meters.consensus_protocol, + parameters.node_parameters.clone(), + ); let node_parameters_string = serde_yaml::to_string(&node_parameters).unwrap(); let node_parameters_path = self.working_dir.join("node-parameters.yaml"); let upload_node_parameters = @@ -256,6 +259,16 @@ impl StarfishProtocol { .join(format!("parameters-{authority}.yaml")) } + fn node_parameters_for_genesis( + consensus_protocol: &str, + mut node_parameters: StarfishNodeParameters, + ) -> StarfishNodeParameters { + if consensus_protocol == "starfish-rbc" { + node_parameters.refresh_starfish_rbc_protocol_instance(); + } + node_parameters + } + fn write_remote_file_command(path: &Path, contents: &str) -> String { let mut delimiter = "STARFISH_REMOTE_FILE_EOF".to_string(); while contents.contains(&delimiter) { @@ -286,7 +299,29 @@ impl StarfishProtocol { #[cfg(test)] mod tests { - use super::StarfishProtocol; + use super::{StarfishNodeParameters, StarfishProtocol}; + + #[test] + fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { + let parameters = StarfishProtocol::node_parameters_for_genesis( + "starfish-rbc", + StarfishNodeParameters::default(), + ); + assert!( + parameters + .starfish_rbc_protocol_instance + .is_some_and(|instance| instance != [0; 32]) + ); + } + + #[test] + fn non_rbc_genesis_does_not_need_a_protocol_instance() { + let parameters = StarfishProtocol::node_parameters_for_genesis( + "starfish", + StarfishNodeParameters::default(), + ); + assert_eq!(parameters.starfish_rbc_protocol_instance, None); + } #[test] fn split_authority_load_preserves_total_load() { diff --git a/crates/starfish-core/src/block_manager.rs b/crates/starfish-core/src/block_manager.rs index 4b34b864..0cbf079f 100644 --- a/crates/starfish-core/src/block_manager.rs +++ b/crates/starfish-core/src/block_manager.rs @@ -65,6 +65,7 @@ impl BlockManager { // missing references that we don't currently have let mut missing_references = AHashSet::new(); let mut block_exists_cache: AHashMap = AHashMap::new(); + let include_ack_dependencies = self.dag_state.consensus_protocol.is_starfish_rbc(); while let Some(block) = blocks.pop_front() { let block_reference = block.reference(); @@ -130,11 +131,11 @@ impl BlockManager { } let mut processed = true; - for included_reference in block.block_references() { - if self.blocks_pending.contains_key(included_reference) { + for included_reference in Self::dependencies(&block, include_ack_dependencies) { + if self.blocks_pending.contains_key(&included_reference) { processed = false; self.block_references_waiting - .entry(*included_reference) + .entry(included_reference) .or_default() .insert(*block_reference); continue; @@ -143,21 +144,21 @@ impl BlockManager { // If we are missing a reference then we insert // into pending and update the waiting index if !*block_exists_cache - .entry(*included_reference) - .or_insert_with(|| self.dag_state.block_exists(*included_reference)) + .entry(included_reference) + .or_insert_with(|| self.dag_state.block_exists(included_reference)) { processed = false; self.block_references_waiting - .entry(*included_reference) + .entry(included_reference) .or_default() .insert(*block_reference); - if !self.blocks_pending.contains_key(included_reference) { + if !self.blocks_pending.contains_key(&included_reference) { // add missing references if it is not available // in both pending set and storage - missing_references.insert(*included_reference); + missing_references.insert(included_reference); self.missing[included_reference.authority as usize] - .insert(*included_reference); + .insert(included_reference); } } } @@ -189,8 +190,7 @@ impl BlockManager { primary key.", ); - if block_pointer - .block_references() + if Self::dependencies(block_pointer, include_ack_dependencies) .iter() .all(|item_ref| !self.block_references_waiting.contains_key(item_ref)) { @@ -220,6 +220,28 @@ impl BlockManager { ) } + /// Dirty-DAG connection normally follows causal parents. Starfish-RBC + /// also waits for logical acknowledgment targets because clean activation + /// treats them as sequencing dependencies and the normal missing-parent + /// request path is the contained way to fetch their headers. + fn dependencies(block: &VerifiedBlock, include_acknowledgments: bool) -> Vec { + let mut seen = AHashSet::new(); + let mut dependencies = Vec::new(); + for reference in block.block_references() { + if seen.insert(*reference) { + dependencies.push(*reference); + } + } + if include_acknowledgments { + for reference in block.acknowledgments() { + if seen.insert(reference) { + dependencies.push(reference); + } + } + } + dependencies + } + pub fn missing_blocks(&self) -> &[AHashSet] { &self.missing } @@ -281,6 +303,28 @@ mod tests { .dag_state } + fn open_rbc_dag_state(committee: Arc, path: &std::path::Path) -> DagState { + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + DagState::open( + 0, + path, + metrics, + committee, + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ) + .dag_state + } + fn make_mac_block( keyrings: &[Vec], authority: AuthorityIndex, @@ -372,4 +416,86 @@ mod tests { .has_full_mac_vector() ); } + + #[test] + fn starfish_rbc_requests_ack_only_dependencies_and_deduplicates_parent_overlap() { + let committee = Committee::new_for_benchmarks(4); + let temp_dir = TempDir::new().unwrap(); + let dag_state = open_rbc_dag_state(committee.clone(), temp_dir.path()); + let mut manager = BlockManager::new(dag_state, &committee); + let genesis: Vec<_> = committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(); + + let mut ack_target = + VerifiedBlock::new_starfish_rbc(1, 1, genesis.clone(), Vec::new(), 1, Vec::new(), None); + ack_target.preserialize(); + let ack_target_ref = *ack_target.reference(); + + // The target is both a parent and an acknowledgment. It must create + // one waiting edge, while the ack-only target below proves that RBC + // expands the dependency set beyond causal parents. + let mut ack_only = + VerifiedBlock::new_starfish_rbc(2, 1, genesis, Vec::new(), 2, Vec::new(), None); + ack_only.preserialize(); + let ack_only_ref = *ack_only.reference(); + let mut parent_a = VerifiedBlock::new_starfish_rbc( + 0, + 1, + committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(), + Vec::new(), + 3, + Vec::new(), + None, + ); + parent_a.preserialize(); + let mut parent_b = VerifiedBlock::new_starfish_rbc( + 3, + 1, + committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(), + Vec::new(), + 4, + Vec::new(), + None, + ); + parent_b.preserialize(); + let parent_a_ref = *parent_a.reference(); + let parent_b_ref = *parent_b.reference(); + manager.add_blocks( + vec![Data::new(parent_a), Data::new(parent_b)], + DataSource::BlockHeaderRequest, + ); + let mut child = VerifiedBlock::new_starfish_rbc( + 0, + 2, + vec![ack_target_ref, parent_a_ref, parent_b_ref], + vec![ack_target_ref, ack_only_ref], + 2, + Vec::new(), + None, + ); + child.preserialize(); + + let (_, _, missing) = + manager.add_blocks(vec![Data::new(child)], DataSource::BlockBundleStreaming); + assert!(missing.contains(&ack_target_ref)); + assert!(missing.contains(&ack_only_ref)); + assert_eq!(manager.pending_blocks_count(), 1); + + manager.add_blocks(vec![Data::new(ack_target)], DataSource::BlockHeaderRequest); + assert_eq!( + manager.pending_blocks_count(), + 1, + "the ack-only dependency must keep the child suspended" + ); + manager.add_blocks(vec![Data::new(ack_only)], DataSource::BlockHeaderRequest); + assert_eq!(manager.pending_blocks_count(), 0); + } } diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index 8db012cd..13de59ed 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -87,6 +87,7 @@ impl BroadcasterParameters { causal_push_shard_round_lag, }, ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::CordialMiners @@ -379,6 +380,7 @@ where match self.inner.dag_state.consensus_protocol { ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::SparseStarfishSpeed => { @@ -432,6 +434,7 @@ where .get_transmission_parts(&refs_to_send, &refs_to_send); let headers = prepare_forwarded_blocks_for_peer( self.inner.dag_state.block_authentication_scheme, + self.inner.dag_state.consensus_protocol, peer_id, headers, ); @@ -473,6 +476,7 @@ where .collect(); let all_blocks = prepare_forwarded_blocks_for_peer( self.inner.dag_state.block_authentication_scheme, + self.inner.dag_state.consensus_protocol, peer_id, all_blocks, ); @@ -929,6 +933,7 @@ struct PushBatchParts { fn push_transport_format(consensus_protocol: ConsensusProtocol) -> PushOtherBlocksFormat { match consensus_protocol { ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::SparseStarfishSpeed => PushOtherBlocksFormat::HeadersAndShards, @@ -1336,6 +1341,7 @@ where .collect(); let other_blocks = prepare_forwarded_blocks_for_peer( inner.dag_state.block_authentication_scheme, + inner.dag_state.consensus_protocol, to_whom_authority_index, other_blocks, ); @@ -1355,6 +1361,7 @@ where .get_transmission_parts(&plan.other_refs, &plan.shard_refs); let headers = prepare_forwarded_blocks_for_peer( inner.dag_state.block_authentication_scheme, + inner.dag_state.consensus_protocol, to_whom_authority_index, headers, ); @@ -1626,9 +1633,12 @@ mod tests { .collect(); let expected = tags[2]; block.header.authentication = BlockAuthentication::MacVector(tags); + let mut rbc_carrier = block.clone(); + rbc_carrier.header.authentication = BlockAuthentication::None; let relayed = prepare_forwarded_blocks_for_peer( BlockAuthenticationScheme::MacVector, + ConsensusProtocol::Starfish, 2, vec![Data::new(block)], ); @@ -1638,8 +1648,24 @@ mod tests { BlockAuthentication::MacTag(tag) if *tag == expected )); - let second_hop = - prepare_forwarded_blocks_for_peer(BlockAuthenticationScheme::MacVector, 3, relayed); + let second_hop = prepare_forwarded_blocks_for_peer( + BlockAuthenticationScheme::MacVector, + ConsensusProtocol::Starfish, + 3, + relayed, + ); assert!(second_hop.is_empty()); + + let forwarded_rbc_carrier = prepare_forwarded_blocks_for_peer( + BlockAuthenticationScheme::MacVector, + ConsensusProtocol::StarfishRbc, + 3, + vec![Data::new(rbc_carrier)], + ); + assert_eq!(forwarded_rbc_carrier.len(), 1); + assert!(matches!( + forwarded_rbc_carrier[0].authentication(), + BlockAuthentication::None + )); } } diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index e453d5ea..34bf365c 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -9,6 +9,7 @@ use std::{ time::Duration, }; +use rand::{RngCore, rngs::OsRng}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ @@ -61,6 +62,11 @@ pub struct NodeParameters { /// protocols select their authentication through the consensus name. #[serde(default)] pub block_authentication: Option, + /// Fresh, nonzero execution identifier shared by every validator in one + /// Starfish-RBC run. It is generated by benchmark setup and ignored by all + /// other protocols. + #[serde(default)] + pub starfish_rbc_protocol_instance: Option<[u8; 32]>, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -135,6 +141,7 @@ impl Default for NodeParameters { bls_verification_workers: node_defaults::default_bls_verification_workers(), dissemination_mode: DisseminationMode::default(), block_authentication: None, + starfish_rbc_protocol_instance: None, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), @@ -150,6 +157,24 @@ impl NodeParameters { ..Self::default() } } + + /// Replace the Starfish-RBC execution identifier with fresh OS randomness. + /// + /// The orchestrator calls this once before serializing the shared node + /// parameters, so independently generated benchmark files on remote hosts + /// still contain the same identifier. + pub fn refresh_starfish_rbc_protocol_instance(&mut self) -> [u8; 32] { + let mut rng = OsRng; + let protocol_instance = loop { + let mut bytes = [0; 32]; + rng.fill_bytes(&mut bytes); + if bytes != [0; 32] { + break bytes; + } + }; + self.starfish_rbc_protocol_instance = Some(protocol_instance); + protocol_instance + } } impl ImportExport for NodeParameters {} @@ -354,6 +379,27 @@ impl NodePrivateConfig { impl ImportExport for NodePrivateConfig {} +#[cfg(test)] +mod tests { + use super::NodeParameters; + + #[test] + fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { + let mut parameters: NodeParameters = serde_yaml::from_str("{}").unwrap(); + assert_eq!(parameters.starfish_rbc_protocol_instance, None); + + let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); + assert_ne!(protocol_instance, [0; 32]); + + let yaml = serde_yaml::to_string(¶meters).unwrap(); + let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!( + decoded.starfish_rbc_protocol_instance, + Some(protocol_instance) + ); + } +} + /// How transaction payloads are filled by the generator. #[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "snake_case")] diff --git a/crates/starfish-core/src/consensus/base_committer.rs b/crates/starfish-core/src/consensus/base_committer.rs index bd47efa0..7f42db59 100644 --- a/crates/starfish-core/src/consensus/base_committer.rs +++ b/crates/starfish-core/src/consensus/base_committer.rs @@ -166,6 +166,7 @@ impl BaseCommitter { let potential_certificates: Vec<_> = certifying_blocks .iter() .filter(|block| reachable.contains(block.reference())) + .filter(|block| self.is_consensus_evidence(block)) .collect(); // Use those potential certificates to determine which (if any) of the target @@ -238,7 +239,10 @@ impl BaseCommitter { ) -> bool { let voting_blocks = self.dag_state.get_blocks_by_round_cached(voting_round); let mut blame_stake_aggregator = StakeAggregator::::new(); - for voting_block in voting_blocks.iter() { + for voting_block in voting_blocks + .iter() + .filter(|block| self.is_consensus_evidence(block)) + { let voter = voting_block.authority(); blame_stake_aggregator.add(voter, &self.committee); } @@ -256,7 +260,10 @@ impl BaseCommitter { for leader_block in &leader_blocks { let mut vote_stake_aggregator = StakeAggregator::::new(); let leader_block_reference = leader_block.reference(); - for voting_block in voting_blocks.iter() { + for voting_block in voting_blocks + .iter() + .filter(|block| self.is_consensus_evidence(block)) + { let voter = voting_block.authority(); if voter_info .voters @@ -292,7 +299,10 @@ impl BaseCommitter { // Quickly reject if there isn't enough stake to support the leader from // the potential certificates. let mut early_stop = true; - for certifying_block in certifying_blocks.iter() { + for certifying_block in certifying_blocks + .iter() + .filter(|block| self.is_consensus_evidence(block)) + { if total_stake_aggregator.add(certifying_block.authority(), &self.committee) { early_stop = false; break; @@ -310,6 +320,7 @@ impl BaseCommitter { self.has_quorum_support( certifying_blocks .iter() + .filter(|block| self.is_consensus_evidence(block)) .filter(|b| self.is_certificate(b, leader_block, voter_info)) .map(|b| b.authority()), ) @@ -419,13 +430,21 @@ impl BaseCommitter { } fn requires_clean_leader_for_commit(&self) -> bool { - self.dag_state.consensus_protocol.is_sailfish_pp() + self.dag_state.consensus_protocol.is_starfish_rbc() + || self.dag_state.consensus_protocol.is_sailfish_pp() || self .dag_state .consensus_protocol .carries_unprovable_certificate() } + /// Starfish-RBC dirty headers exist only for dependency fetching and RBC + /// progress. They cannot vote, blame, or certify a leader. + fn is_consensus_evidence(&self, block: &VerifiedBlock) -> bool { + !self.dag_state.consensus_protocol.is_starfish_rbc() + || self.dag_state.has_clean_vertex(block.reference()) + } + /// Apply the indirect decision rule to the specified leader /// to see whether we can indirect-commit or indirect-skip it. #[tracing::instrument(skip_all, fields(leader = %format_authority_round(leader, leader_round)))] @@ -527,3 +546,97 @@ impl Display for BaseCommitter { write!(f, "Committer-Round-Offset{}", self.options.round_offset) } } + +#[cfg(test)] +mod tests { + use ahash::{AHashMap, AHashSet}; + use prometheus::Registry; + use tempfile::TempDir; + + use super::*; + use crate::{ + config::{DisseminationMode, StorageBackend}, + dag_state::DataSource, + metrics::Metrics, + }; + + #[test] + fn starfish_rbc_dirty_voters_cannot_force_a_skip() { + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let dag_state = DagState::open( + 0, + dir.path(), + metrics, + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ) + .dag_state; + let genesis: Vec<_> = committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(); + let mut parent_blocks = Vec::new(); + for authority in [0, 1, 2] { + let mut block = VerifiedBlock::new_starfish_rbc( + authority, + 1, + genesis.clone(), + Vec::new(), + authority as u64, + Vec::new(), + None, + ); + block.preserialize(); + parent_blocks.push(Data::new(block)); + } + let parent_refs: Vec<_> = parent_blocks + .iter() + .map(|block| *block.reference()) + .collect(); + dag_state.insert_general_blocks(parent_blocks, DataSource::BlockBundleStreaming); + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&parent_refs)); + + let mut voting_blocks = Vec::new(); + for authority in [0, 2, 3] { + let mut block = VerifiedBlock::new_starfish_rbc( + authority, + 2, + parent_refs.clone(), + Vec::new(), + authority as u64, + Vec::new(), + None, + ); + block.preserialize(); + voting_blocks.push(Data::new(block)); + } + let voting_refs: Vec<_> = voting_blocks + .iter() + .map(|block| *block.reference()) + .collect(); + dag_state.insert_general_blocks(voting_blocks, DataSource::BlockBundleStreaming); + + let committer = BaseCommitter::new(committee.clone(), dag_state.clone()); + let voter_info = VoterInfo { + voters: AHashSet::new(), + voter_strong_votes: AHashMap::new(), + }; + let leader = committee.elect_leader(1); + assert!(!committer.decide_skip(2, leader, &voter_info)); + + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&voting_refs)); + assert!(committer.decide_skip(2, leader, &voter_info)); + } +} diff --git a/crates/starfish-core/src/consensus/linearizer.rs b/crates/starfish-core/src/consensus/linearizer.rs index 3a624b8e..32f8cde2 100644 --- a/crates/starfish-core/src/consensus/linearizer.rs +++ b/crates/starfish-core/src/consensus/linearizer.rs @@ -219,6 +219,7 @@ impl Linearizer { tracing::debug!("Starting collection with leader {:?}", leader_block); let leader_block_ref = *leader_block.reference(); let min_round = leader_block_ref.round.saturating_sub(MAX_TRAVERSAL_DEPTH); + let clean_only = dag_state.consensus_protocol.is_starfish_rbc(); let mut committed_ack_refs = BTreeSet::new(); let mut current_level = vec![leader_block]; @@ -228,11 +229,24 @@ impl Linearizer { while !current_level.is_empty() { let mut next_refs = Vec::new(); for x in ¤t_level { + if clean_only && !dag_state.has_clean_vertex(x.reference()) { + tracing::warn!( + "Ignoring dirty Starfish-RBC block {} at the linearization boundary", + x.reference() + ); + continue; + } let who_votes = x.authority(); for ack_ref in self.effective_acknowledgments(dag_state, x) { if ack_ref.round < min_round { continue; } + if clean_only && !dag_state.has_clean_vertex(&ack_ref) { + tracing::warn!( + "Ignoring dirty Starfish-RBC acknowledgment target {ack_ref}" + ); + continue; + } if direct_ack { if ack_ref.authority != x.authority() { continue; @@ -247,7 +261,10 @@ impl Linearizer { } self.traversed_blocks.insert(*x.reference()); for reference in x.block_references() { - if reference.round >= min_round && self.traversed_blocks.insert(*reference) { + if reference.round >= min_round + && (!clean_only || dag_state.has_clean_vertex(reference)) + && self.traversed_blocks.insert(*reference) + { next_refs.push(*reference); } } @@ -265,6 +282,7 @@ impl Linearizer { // Phase 2: batch-fetch the newly committed ack refs. let new_ack_refs: Vec<_> = committed_ack_refs .into_iter() + .filter(|reference| !clean_only || dag_state.has_clean_vertex(reference)) .filter(|r| self.committed.insert(*r)) .collect(); @@ -360,6 +378,15 @@ impl Linearizer { let consensus_protocol = dag_state.consensus_protocol; let mut committed = vec![]; for (leader_block, metastate) in committed_leaders { + if consensus_protocol.is_starfish_rbc() + && !dag_state.has_clean_vertex(leader_block.reference()) + { + tracing::warn!( + "Ignoring dirty Starfish-RBC leader {} at the linearization boundary", + leader_block.reference() + ); + continue; + } // Collect the sub-dag generated using each of these leaders as anchor. let leader_ref = *leader_block.reference(); let leader_acks = leader_block.acknowledgments(); @@ -370,6 +397,7 @@ impl Linearizer { self.collect_subdag_acknowledgments(dag_state, leader_block, true) } ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed => { self.collect_subdag_acknowledgments(dag_state, leader_block, false) @@ -502,6 +530,25 @@ mod tests { Data::new(block) } + fn make_rbc_block( + authority: AuthorityIndex, + round: RoundNumber, + parents: Vec, + acks: Vec, + ) -> Data { + let mut block = VerifiedBlock::new_starfish_rbc( + authority, + round, + parents, + acks, + round as u64, + Vec::new(), + None, + ); + block.preserialize(); + Data::new(block) + } + #[test] fn ssfs_opt_sequences_leader_and_leader_acknowledgments() { let (committee, dag_state) = open_test_dag_state_for("sparse-starfish-speed"); @@ -587,4 +634,52 @@ mod tests { "Std must not directly sequence leader acknowledgments" ); } + + #[test] + fn starfish_rbc_linearizer_ignores_dirty_history_and_sequences_clean_quorum_acks() { + let (committee, dag_state) = open_test_dag_state_for("starfish-rbc"); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let target = make_rbc_block(0, 1, genesis.clone(), Vec::new()); + let parent_a = make_rbc_block(1, 1, genesis.clone(), Vec::new()); + let parent_b = make_rbc_block(2, 1, genesis, Vec::new()); + let target_ref = *target.reference(); + let round_one_refs = vec![target_ref, *parent_a.reference(), *parent_b.reference()]; + let voters: Vec<_> = (0..3) + .map(|authority| make_rbc_block(authority, 2, round_one_refs.clone(), vec![target_ref])) + .collect(); + let voter_refs: Vec<_> = voters.iter().map(|block| *block.reference()).collect(); + let leader = make_rbc_block(3, 3, voter_refs.clone(), Vec::new()); + let leader_ref = *leader.reference(); + let mut all_blocks = vec![target, parent_a, parent_b]; + all_blocks.extend(voters); + all_blocks.push(leader.clone()); + dag_state.insert_general_blocks(all_blocks, DataSource::BlockBundleStreaming); + + let mut dirty_linearizer = Linearizer::new(committee.clone()); + assert!( + dirty_linearizer + .handle_commit(&dag_state, vec![(leader.clone(), None)]) + .is_empty(), + "a dirty leader cannot enter the Starfish-RBC linearizer" + ); + + let mut delivered = round_one_refs; + delivered.extend(voter_refs); + delivered.push(leader_ref); + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&delivered)); + + let mut clean_linearizer = Linearizer::new(committee); + let committed = clean_linearizer.handle_commit(&dag_state, vec![(leader, None)]); + assert_eq!(committed.len(), 1); + assert!( + committed[0] + .0 + .blocks + .iter() + .any(|block| block.reference() == &target_ref), + "only clean voting headers may form acknowledgment quorum" + ); + } } diff --git a/crates/starfish-core/src/consensus/universal_committer.rs b/crates/starfish-core/src/consensus/universal_committer.rs index 6ea87d25..7602b00e 100644 --- a/crates/starfish-core/src/consensus/universal_committer.rs +++ b/crates/starfish-core/src/consensus/universal_committer.rs @@ -88,6 +88,11 @@ impl UniversalCommitter { let mut voters = AHashSet::new(); let mut voter_strong_votes = AHashMap::new(); for vb in potential_voting_blocks.iter() { + if self.dag_state.consensus_protocol.is_starfish_rbc() + && !self.dag_state.has_clean_vertex(vb.reference()) + { + continue; + } let vb_ref = *vb.reference(); if self.dag_state.consensus_protocol.uses_bls() { if let Some(leader_ref) = @@ -404,6 +409,7 @@ impl UniversalCommitterBuilder { match dag_state.consensus_protocol { ConsensusProtocol::Mysticeti | ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::MysticetiBls diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index abfeeced..82dc5009 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -215,6 +215,14 @@ impl Core { &self.signer } + pub(crate) fn get_ml_dsa_44_signer(&self) -> &crate::crypto::MlDsa44Signer { + &self.ml_dsa_44_signer + } + + pub(crate) fn get_ml_dsa_65_signer(&self) -> &crate::crypto::MlDsa65Signer { + &self.ml_dsa_65_signer + } + pub fn mac_keys(&self) -> Arc> { self.mac_keys.clone() } @@ -511,6 +519,24 @@ impl Core { return None; } + // `build_block` always prepends the creator's previous block. For + // Starfish-RBC that local header is dirty until the local RBC instance + // delivers it; another clean quorum must not let us smuggle this dirty + // mandatory parent into a proposal. + if protocol.is_starfish_rbc() + && clock_round > 1 + && self + .last_own_block + .iter() + .any(|own| !self.dag_state.has_clean_vertex(own.block.reference())) + { + tracing::debug!( + "Cannot construct Starfish-RBC block in round {}: own previous header is not clean", + clock_round + ); + return None; + } + let voted_leader_ref = if protocol.uses_bls() { self.select_starfish_bls_voted_leader(clock_round) } else { @@ -531,8 +557,17 @@ impl Core { }; let pending_transactions = self.get_pending_transactions(clock_round); - let (mut transactions, block_references, raw_refs) = + let (mut transactions, block_references, raw_refs, deferred_dirty_refs) = self.collect_transactions_and_references(pending_transactions, clock_round); + // A header can reach the dirty DAG before the local RBC instance + // delivers it. Keep its include notification pending so a proposal + // created from some other clean quorum does not permanently consume + // the only chance to reference it once delivery completes. + self.pending.extend( + deferred_dirty_refs + .into_iter() + .map(MetaTransaction::Include), + ); // Dual-DAG protocols: if the clean-parent filter reduced the parent // set below threshold-clock quorum, we cannot build a valid block yet. @@ -713,6 +748,7 @@ impl Core { Vec, Vec, Vec, + Vec, ) { let mut transactions = Vec::new(); let mut pending_refs = Vec::new(); @@ -724,9 +760,23 @@ impl Core { MetaTransaction::Include(include) => pending_refs.push(include), } } - let raw_refs = pending_refs.clone(); + // Dirty vertices must not even participate in transitive reduction: + // otherwise a dirty child can suppress one of its clean parents and + // then be filtered itself, shrinking the usable clean frontier. + let (compression_candidates, deferred_dirty_refs): (Vec<_>, Vec<_>) = + if self.dag_state.consensus_protocol.is_starfish_rbc() { + pending_refs.into_iter().partition(|reference| { + reference.round == 0 || self.dag_state.has_clean_vertex(reference) + }) + } else { + (pending_refs, Vec::new()) + }; + // These are the usable inputs that callers must retry when a later + // proposal gate fails. Dirty RBC refs are retried independently above + // so the two retry paths cannot duplicate them. + let raw_refs = compression_candidates.clone(); let mut block_references = - self.compress_pending_block_references(&pending_refs, block_round); + self.compress_pending_block_references(&compression_candidates, block_round); // Dual-DAG protocols: filter parents to only include clean blocks. if self.dag_state.consensus_protocol.uses_dual_dag() { @@ -790,11 +840,16 @@ impl Core { seen.contains(self.authority), is_compressed_non_leader ); - return (transactions, vec![], raw_refs); + return (transactions, vec![], raw_refs, deferred_dirty_refs); } } - (transactions, block_references, raw_refs) + ( + transactions, + block_references, + raw_refs, + deferred_dirty_refs, + ) } fn prepare_encoded_transactions( @@ -1004,34 +1059,50 @@ impl Core { None }; - let authorizer = match self.dag_state.block_authentication_scheme { - BlockAuthenticationScheme::Ed25519 => BlockAuthorizer::Ed25519(&self.signer), - BlockAuthenticationScheme::MacVector => BlockAuthorizer::MacVector(&self.mac_keys), - BlockAuthenticationScheme::MlDsa44 => BlockAuthorizer::MlDsa44(&self.ml_dsa_44_signer), - BlockAuthenticationScheme::MlDsa65 => BlockAuthorizer::MlDsa65(&self.ml_dsa_65_signer), + let mut block = if protocol == ConsensusProtocol::StarfishRbc { + VerifiedBlock::new_starfish_rbc( + self.authority, + clock_round, + block_references, + acknowledgment_references.to_vec(), + time_ns, + transactions.to_vec(), + encoded_transactions.clone(), + ) + } else { + let authorizer = match self.dag_state.block_authentication_scheme { + BlockAuthenticationScheme::Ed25519 => BlockAuthorizer::Ed25519(&self.signer), + BlockAuthenticationScheme::MacVector => BlockAuthorizer::MacVector(&self.mac_keys), + BlockAuthenticationScheme::MlDsa44 => { + BlockAuthorizer::MlDsa44(&self.ml_dsa_44_signer) + } + BlockAuthenticationScheme::MlDsa65 => { + BlockAuthorizer::MlDsa65(&self.ml_dsa_65_signer) + } + }; + VerifiedBlock::new_with_authorizer_and_unprovable( + self.authority, + clock_round, + block_references, + voted_leader_ref, + acknowledgment_references.to_vec(), + time_ns, + &authorizer, + bls_signer_opt, + committee_opt, + aggregate_dac_sigs, + transactions.to_vec(), + encoded_transactions.clone(), + self.dag_state.consensus_protocol, + strong_vote, + aggregate_round_sig, + certified_leader, + precomputed_round_sig, + precomputed_leader_sig, + sailfish_fields, + unprovable_certificate, + ) }; - let mut block = VerifiedBlock::new_with_authorizer_and_unprovable( - self.authority, - clock_round, - block_references, - voted_leader_ref, - acknowledgment_references.to_vec(), - time_ns, - &authorizer, - bls_signer_opt, - committee_opt, - aggregate_dac_sigs, - transactions.to_vec(), - encoded_transactions.clone(), - self.dag_state.consensus_protocol, - strong_vote, - aggregate_round_sig, - certified_leader, - precomputed_round_sig, - precomputed_leader_sig, - sailfish_fields, - unprovable_certificate, - ); let role = if is_round_leader { "leader" @@ -1627,6 +1698,26 @@ mod tests { Data::new(block) } + fn make_starfish_rbc_round_1_block( + committee: &Committee, + authority: AuthorityIndex, + ) -> Data { + let mut block = VerifiedBlock::new_starfish_rbc( + authority, + 1, + committee + .authorities() + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(), + Vec::new(), + authority as u64, + Vec::new(), + None, + ); + block.preserialize(); + Data::new(block) + } + fn make_test_round_certificate( bls_signers: &[BlsSigner], round: RoundNumber, @@ -1707,6 +1798,69 @@ mod tests { assert_eq!(refs[0], *round_1.reference()); } + #[test] + fn starfish_rbc_proposal_defers_dirty_include_until_delivery() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let private_config = NodePrivateConfig::new_for_tests(authority); + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + private_config, + metrics, + recovered, + None, + ); + + let own_round_one = core + .try_new_block("new_blocks") + .expect("round-one block should be creatable"); + let peer_one = make_starfish_rbc_round_1_block(&committee, 1); + let peer_two = make_starfish_rbc_round_1_block(&committee, 2); + let dirty_peer = make_starfish_rbc_round_1_block(&committee, 3); + let dirty_ref = *dirty_peer.reference(); + core.add_headers( + vec![peer_one.clone(), peer_two.clone(), dirty_peer], + DataSource::BlockBundleStreamingHeader, + ); + + assert!( + core.dag_state() + .apply_starfish_rbc_delivery_refs_for_test(&[ + *own_round_one.reference(), + *peer_one.reference(), + *peer_two.reference(), + ]) + ); + let round_two = core + .try_new_block("new_blocks") + .expect("a clean quorum should permit the round-two proposal"); + assert!(!round_two.block_references().contains(&dirty_ref)); + assert!(core.pending.iter().any(|pending| { + matches!(pending, MetaTransaction::Include(reference) if *reference == dirty_ref) + })); + } + #[test] fn mysticeti_bls_non_leader_can_build_round_2_with_prev_leader_parent() { let authority = 0; diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index aefcdb78..7324c25f 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -13,6 +13,7 @@ use crate::{ dag_state::DataSource, data::Data, metrics::{Metrics, UtilizationTimerExt}, + starfish_rbc::PinnedRbcHeader, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ AuthorityIndex, BlockReference, ProvableShard, ReconstructedTransactionData, RoundNumber, @@ -68,6 +69,8 @@ enum CoreThreadCommand { ApplyCertificateEvents(Vec, oneshot::Sender<()>), /// Apply Sailfish RBC-certified vertices on the core thread. ApplySailfishCertificates(Vec, oneshot::Sender<()>), + /// Apply locally delivered Starfish-RBC headers on the core thread. + ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), /// Store a Sailfish++ timeout certificate in DagState. ApplyTimeoutCert(SailfishTimeoutCert, oneshot::Sender<()>), /// Store a Sailfish++ no-vote certificate in DagState. @@ -192,6 +195,22 @@ impl, + ) { + let (sender, receiver) = oneshot::channel(); + self.send(CoreThreadCommand::ApplyStarfishRbcDeliveries( + delivered_headers, + sender, + )) + .await; + receiver.await.expect("core thread is not expected to stop"); + } + /// Store a Sailfish++ timeout certificate on the core thread. pub async fn apply_timeout_cert(&self, cert: SailfishTimeoutCert) { let (sender, receiver) = oneshot::channel(); @@ -375,6 +394,14 @@ impl CoreThread { self.syncer.apply_sailfish_certificates(certified_refs); sender.send(()).ok(); } + CoreThreadCommand::ApplyStarfishRbcDeliveries(delivered_headers, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_deliveries"]) + .inc(); + self.syncer.apply_starfish_rbc_deliveries(delivered_headers); + sender.send(()).ok(); + } CoreThreadCommand::ApplyTimeoutCert(cert, sender) => { metrics .core_thread_tasks_total @@ -477,7 +504,7 @@ mod tests { recovered, None, ); - let syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None); + let syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None, None); CoreThreadDispatcher::start(syncer) } diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 602ecafb..da4903ea 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -28,6 +28,7 @@ use crate::{ metrics::{Metrics, UtilizationTimerExt}, network::ShardPayload, rocks_store::RocksStore, + starfish_rbc::PinnedRbcHeader, state::{RecoveredState, RecoveredStateBuilder}, store::Store, threshold_clock::ThresholdClockAggregator, @@ -118,6 +119,9 @@ pub enum ConsensusProtocol { Mysticeti, CordialMiners, Starfish, + /// Plain Starfish ordering over headers certified by the Starfish-RBC + /// reliable-broadcast service. + StarfishRbc, StarfishSpeed, StarfishBls, SailfishPlusPlus, @@ -177,6 +181,7 @@ impl ConsensusProtocol { "mysticeti" => Some(ConsensusProtocol::Mysticeti), "cordial-miners" => Some(ConsensusProtocol::CordialMiners), "starfish" => Some(ConsensusProtocol::Starfish), + "starfish-rbc" => Some(ConsensusProtocol::StarfishRbc), "starfish-bls" | "starfish-l" => Some(ConsensusProtocol::StarfishBls), "starfish-speed" | "starfish-s" => Some(ConsensusProtocol::StarfishSpeed), "sailfish++" | "sailfish-pp" => Some(ConsensusProtocol::SailfishPlusPlus), @@ -193,6 +198,7 @@ impl ConsensusProtocol { matches!( self, ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishBls | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed @@ -203,6 +209,10 @@ impl ConsensusProtocol { matches!(self, ConsensusProtocol::SailfishPlusPlus) } + pub fn is_starfish_rbc(self) -> bool { + matches!(self, ConsensusProtocol::StarfishRbc) + } + pub fn is_bluestreak(self) -> bool { matches!(self, ConsensusProtocol::Bluestreak) } @@ -243,6 +253,7 @@ impl ConsensusProtocol { matches!( self, ConsensusProtocol::SailfishPlusPlus + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::Bluestreak | ConsensusProtocol::StarfishBls | ConsensusProtocol::MysticetiBls @@ -268,6 +279,7 @@ impl ConsensusProtocol { | ConsensusProtocol::StarfishBls => DisseminationMode::Pull, ConsensusProtocol::CordialMiners => DisseminationMode::PushCausal, ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed => DisseminationMode::PushUseful, } @@ -341,10 +353,20 @@ impl ProtocolConfig { "ed25519" => BlockAuthenticationScheme::Ed25519, "ml-dsa-44" => BlockAuthenticationScheme::MlDsa44, "ml-dsa-65" => BlockAuthenticationScheme::MlDsa65, + "mac" if consensus_protocol.is_starfish_rbc() => { + BlockAuthenticationScheme::MacVector + } + "mac" => { + return Err( + "MAC initial authentication is only available for 'starfish-rbc'; use an \ + experimental '*-mac' protocol for the legacy lower-bound benchmark" + .to_string(), + ); + } value => { return Err(format!( "Unknown block authentication scheme '{value}'. Use 'ed25519', \ - 'ml-dsa-44', or 'ml-dsa-65'." + 'ml-dsa-44', or 'ml-dsa-65' (and 'mac' for 'starfish-rbc')." )); } } @@ -527,6 +549,14 @@ struct DagStateInner { /// A vertex joins the clean DAG only once its direct parents are also /// clean (or genesis), making the usable clean DAG ancestor-closed. clean_vertices: Vec>, + /// Starfish-RBC headers that this validator has locally delivered. + /// + /// Delivery and dirty-DAG insertion are independent asynchronous events: + /// a delivered header may still be waiting in `BlockManager` for a + /// missing parent. Retaining this exact-reference latch makes either + /// event order safe. Version one deliberately keeps these latches for the + /// process lifetime because no RBC retirement watermark has been proved. + rbc_delivered_vertices: Vec>, /// Per-round support (by stake) for clean vertices. Tracks which /// authorities have at least one clean vertex at a given round so we can /// answer clean-quorum checks without scanning all authorities. @@ -683,6 +713,7 @@ impl DagState { precomputed_round_sigs: BTreeMap::new(), precomputed_leader_sigs: BTreeMap::new(), clean_vertices: (0..n).map(|_| BTreeSet::new()).collect(), + rbc_delivered_vertices: (0..n).map(|_| BTreeSet::new()).collect(), clean_round_support: BTreeMap::new(), clean_quorum_round: 0, pending_clean_vertices: (0..n).map(|_| BTreeSet::new()).collect(), @@ -900,6 +931,7 @@ impl DagState { match &consensus_protocol { ConsensusProtocol::Mysticeti => tracing::info!("Starting Mysticeti protocol"), ConsensusProtocol::Starfish => tracing::info!("Starting Starfish protocol"), + ConsensusProtocol::StarfishRbc => tracing::info!("Starting Starfish-RBC protocol"), ConsensusProtocol::StarfishBls => tracing::info!("Starting Starfish-BLS protocol"), ConsensusProtocol::StarfishSpeed => tracing::info!("Starting Starfish-Speed protocol"), ConsensusProtocol::CordialMiners => tracing::info!("Starting Cordial Miners protocol"), @@ -1416,6 +1448,17 @@ impl DagState { inner.precomputed_leader_sigs.insert(leader_ref, sig); } CertificateEvent::BlockVerified(block_ref) => { + // This event is a capability produced only by the BLS + // verifier. In particular it must never become an + // alternate clean-admission path for Starfish-RBC, whose + // sole production capability is `PinnedRbcHeader`. + if !inner.consensus_protocol.uses_bls() { + tracing::warn!( + "Ignoring BLS block-verification event for non-BLS protocol: {}", + block_ref + ); + continue; + } let auth = block_ref.authority as usize; if inner.get_block(block_ref).is_none() { inner.bls_verified_blocks[auth].insert(block_ref); @@ -1478,6 +1521,10 @@ impl DagState { /// A newly delivered vertex becomes usable only after all of its direct /// parents are also clean (or genesis). pub fn mark_vertices_clean(&self, block_refs: &[BlockReference]) -> bool { + assert!( + !self.consensus_protocol.is_starfish_rbc(), + "Starfish-RBC cleanliness requires a local RBC delivery event" + ); if block_refs.is_empty() { return false; } @@ -1497,6 +1544,42 @@ impl DagState { self.mark_vertices_clean(&[block_ref]) } + /// Apply locally observed Starfish-RBC deliveries. + /// + /// The pin is the service-to-core delivery capability. The delivery latch + /// is recorded even when the header is not dirty-DAG connected yet; + /// `add_block` completes activation after insertion. + pub(crate) fn apply_starfish_rbc_deliveries(&self, delivered: &[PinnedRbcHeader]) -> bool { + assert!( + self.consensus_protocol.is_starfish_rbc(), + "RBC deliveries are only valid for Starfish-RBC" + ); + if delivered.is_empty() { + return false; + } + + let mut inner = self.dag_state_inner.write(); + let mut activated = Vec::new(); + for header in delivered { + inner.record_rbc_delivery(header.reference(), &self.committee, &mut activated); + } + !activated.is_empty() + } + + #[cfg(test)] + pub(crate) fn apply_starfish_rbc_delivery_refs_for_test( + &self, + delivered: &[BlockReference], + ) -> bool { + assert!(self.consensus_protocol.is_starfish_rbc()); + let mut inner = self.dag_state_inner.write(); + let mut activated = Vec::new(); + for &block_ref in delivered { + inner.record_rbc_delivery(block_ref, &self.committee, &mut activated); + } + !activated.is_empty() + } + /// Drain clean dual-DAG vertices that still need to be persisted at /// the next storage flush boundary. pub fn take_pending_clean_refs(&self) -> Vec { @@ -1865,7 +1948,12 @@ impl DagState { ) -> bool { let inner = self.dag_state_inner.read(); let leader_round = quorum_round - 1; - let blocks = inner.get_blocks_by_round(leader_round); + let mut blocks = inner.get_blocks_by_round(leader_round); + if self.consensus_protocol.is_starfish_rbc() { + blocks.retain(|block| { + inner.clean_vertices[block.authority() as usize].contains(block.reference()) + }); + } if blocks.is_empty() { return false; } @@ -2446,6 +2534,15 @@ impl DagState { pub fn cleanup(&self) { let _timer = self.metrics.dag_state_cleanup_util.utilization_timer(); + // Version-one Starfish-RBC deliberately has no state-retirement + // rule. Phase locks, delivered slots, clean dependencies, and the + // blocks consumed by clean-only consensus must remain available for + // the process lifetime. A bounded GC requires a proved durable + // retirement watermark and is deferred with crash recovery. + if self.consensus_protocol.is_starfish_rbc() { + return; + } + let (highest_round, lowest_round, block_count, max_evicted, evicted_rounds) = { let mut inner = self.dag_state_inner.write(); inner.evict_per_authority(); @@ -2867,6 +2964,24 @@ impl DagStateInner { } } + /// Record local reliable delivery and activate the exact vertex if its + /// dirty-DAG carrier is already present. If insertion is still blocked on + /// a missing parent, `add_block` observes the retained latch later. + fn record_rbc_delivery( + &mut self, + block_ref: BlockReference, + committee: &Committee, + activated: &mut Vec, + ) { + let auth = block_ref.authority as usize; + if !self.rbc_delivered_vertices[auth].insert(block_ref) { + return; + } + if self.get_block(block_ref).is_some() { + self.note_clean_vertex(block_ref, committee, activated); + } + } + /// Register a locally verified dual-DAG vertex. /// If all causal predecessors are already clean, the vertex activates /// immediately; otherwise it waits on the missing clean dependencies. @@ -2883,51 +2998,60 @@ impl DagStateInner { return; } - let missing_parents = self.missing_clean_parents(block_ref); - if missing_parents.is_empty() { + let missing_dependencies = self.missing_clean_dependencies(block_ref); + if missing_dependencies.is_empty() { self.activate_clean_vertex(block_ref, committee, activated); return; } self.pending_clean_vertices[auth].insert(block_ref); self.pending_clean_vertex_counts - .insert(block_ref, missing_parents.len()); - for parent in missing_parents { + .insert(block_ref, missing_dependencies.len()); + for dependency in missing_dependencies { self.pending_clean_vertex_children - .entry(parent) + .entry(dependency) .or_default() .insert(block_ref); } } - /// Return the direct causal dependencies that still block this vertex from - /// entering the clean DAG. - fn missing_clean_parents(&self, block_ref: BlockReference) -> Vec { + /// Return the exact clean dependencies that still block this vertex. + /// Starfish-RBC includes logical acknowledgments because they can affect + /// sequencing. A reference shared by the parent and compressed-ack lists + /// is counted once; otherwise the reverse BTreeSet would wake the child + /// once while its missing count remained above zero. + fn missing_clean_dependencies(&self, block_ref: BlockReference) -> Vec { let block = self .get_storage_block(block_ref) .unwrap_or_else(|| panic!("Clean block {block_ref} should exist in DagState")); - let mut missing = Vec::new(); + let mut dependencies = BTreeSet::new(); for parent in block.block_references() { - if parent.round == 0 { - continue; + if parent.round > 0 { + dependencies.insert(*parent); } - if !self.clean_vertices[parent.authority as usize].contains(parent) { - missing.push(*parent); + } + if self.consensus_protocol.is_starfish_rbc() { + for acknowledgment in block.acknowledgments() { + if acknowledgment.round > 0 { + dependencies.insert(acknowledgment); + } } } // Bluestreak / SparseStarfishSpeed: the unprovable_certificate target // is also a causal dependency that must be ancestor-closed. if self.consensus_protocol.carries_unprovable_certificate() { if let Some((cert_ref, _strong)) = block.unprovable_certificate() { - if cert_ref.round > 0 - && !self.clean_vertices[cert_ref.authority as usize].contains(&cert_ref) - && !missing.contains(&cert_ref) - { - missing.push(cert_ref); + if cert_ref.round > 0 { + dependencies.insert(cert_ref); } } } - missing + dependencies + .into_iter() + .filter(|dependency| { + !self.clean_vertices[dependency.authority as usize].contains(dependency) + }) + .collect() } /// Move a vertex into the clean DAG and recursively wake any children that @@ -2951,11 +3075,25 @@ impl DagStateInner { let block = self .get_storage_block(block_ref) .unwrap_or_else(|| panic!("Clean block {block_ref} should exist in DagState")); - for parent in block.block_references() { - if let Some(children) = self.pending_clean_vertex_children.get_mut(parent) { + let mut dependencies: BTreeSet = block + .block_references() + .iter() + .copied() + .filter(|reference| reference.round > 0) + .collect(); + if self.consensus_protocol.is_starfish_rbc() { + dependencies.extend( + block + .acknowledgments() + .into_iter() + .filter(|reference| reference.round > 0), + ); + } + for dependency in dependencies { + if let Some(children) = self.pending_clean_vertex_children.get_mut(&dependency) { children.remove(&block_ref); if children.is_empty() { - self.pending_clean_vertex_children.remove(parent); + self.pending_clean_vertex_children.remove(&dependency); } } } @@ -2973,6 +3111,14 @@ impl DagStateInner { activated.push(block_ref); self.pending_persisted_clean_vertices[auth].insert(block_ref); + if self.consensus_protocol.is_starfish_rbc() { + // Invalidate the universal committer's cached voter view when an + // already-inserted block becomes consensus-visible. + *self.round_version.entry(block_ref.round).or_insert(0) += 1; + if !block.has_empty_payload() { + self.maybe_queue_ack(block_ref); + } + } let waiting_children = self .pending_clean_vertex_children @@ -3041,7 +3187,12 @@ impl DagStateInner { // For SSFS this also gives `compute_unprovable_certificate` an O(1) // precomputed strong-quorum lookup at block creation. self.check_pre_clean(&block, committee, activated); - if self.bls_verified_blocks[auth].remove(reference) { + if self.consensus_protocol.is_starfish_rbc() + && self.rbc_delivered_vertices[auth].contains(reference) + { + self.note_clean_vertex(*reference, committee, activated); + } + if self.consensus_protocol.uses_bls() && self.bls_verified_blocks[auth].remove(reference) { self.note_clean_vertex(*reference, committee, activated); } } @@ -3056,7 +3207,10 @@ impl DagStateInner { committee: &Committee, activated: &mut Vec, ) { - if !self.consensus_protocol.uses_dual_dag() || block.round() <= 1 { + if !self.consensus_protocol.uses_dual_dag() + || self.consensus_protocol.is_starfish_rbc() + || block.round() <= 1 + { return; } @@ -3322,21 +3476,31 @@ impl DagStateInner { /// Queue an acknowledgment for `block_ref` only when all prerequisites are /// met. For StarfishBls the block must be both data-available and - /// DAC-certified; other protocols only require data availability. + /// DAC-certified. Starfish-RBC additionally requires local clean + /// activation, so dirty headers can never influence acknowledgment-based + /// sequencing. fn maybe_queue_ack(&mut self, block_ref: BlockReference) { - let Some(pending) = self.pending_acknowledgment.as_mut() else { - return; - }; let auth = block_ref.authority as usize; if !self.data_availability[auth].contains(&block_ref) { return; } + if self.consensus_protocol.is_starfish_rbc() + && !self.clean_vertices[auth].contains(&block_ref) + { + return; + } if self.consensus_protocol == ConsensusProtocol::StarfishBls && (block_ref.authority != self.authority || !self.dac_certificates[auth].contains_key(&block_ref)) { return; } + let Some(pending) = self.pending_acknowledgment.as_mut() else { + return; + }; + if self.consensus_protocol.is_starfish_rbc() && pending.contains(&block_ref) { + return; + } pending.push(block_ref); } @@ -3566,12 +3730,13 @@ mod tests { TransactionsCommitment, }, data::Data, + encoder::ShardEncoder, metrics::Metrics, types::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthentication, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, - ProvableShard, RoundNumber, SailfishFields, SailfishNoVoteCert, Transaction, - VerifiedBlock, + Encoder, ProvableShard, RoundNumber, SailfishFields, SailfishNoVoteCert, Transaction, + TransactionData, VerifiedBlock, }, }; @@ -3756,11 +3921,31 @@ mod tests { Data::new(block) } + fn make_starfish_rbc_block( + authority: AuthorityIndex, + round: RoundNumber, + parents: Vec, + acknowledgments: Vec, + ) -> Data { + let mut block = VerifiedBlock::new_starfish_rbc( + authority, + round, + parents, + acknowledgments, + round as u64, + Vec::new(), + None, + ); + block.preserialize(); + Data::new(block) + } + #[test] fn acknowledgments_are_only_enabled_for_starfish_variants() { assert!(!ConsensusProtocol::Mysticeti.supports_acknowledgments()); assert!(!ConsensusProtocol::CordialMiners.supports_acknowledgments()); assert!(ConsensusProtocol::Starfish.supports_acknowledgments()); + assert!(ConsensusProtocol::StarfishRbc.supports_acknowledgments()); assert!(ConsensusProtocol::StarfishSpeed.supports_acknowledgments()); assert!(ConsensusProtocol::StarfishBls.supports_acknowledgments()); assert!(ConsensusProtocol::SparseStarfishSpeed.supports_acknowledgments()); @@ -3958,6 +4143,222 @@ mod tests { assert!(dag_state.has_clean_vertex(&child_ref)); } + #[test] + fn starfish_rbc_delivery_is_order_independent_and_waits_for_ack_closure() { + let dag_state = open_test_dag_state_for("starfish-rbc", 0); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let parent_a = make_starfish_rbc_block(0, 1, genesis.clone(), Vec::new()); + let parent_b = make_starfish_rbc_block(1, 1, genesis.clone(), Vec::new()); + let parent_c = make_starfish_rbc_block(2, 1, genesis, Vec::new()); + let parent_a_ref = *parent_a.reference(); + let parent_b_ref = *parent_b.reference(); + let parent_c_ref = *parent_c.reference(); + // parent_b is both a causal parent and a compressed logical ack. It + // must contribute only one missing dependency/wakeup. + let child = make_starfish_rbc_block( + 3, + 2, + vec![parent_a_ref, parent_b_ref, parent_c_ref], + vec![parent_b_ref], + ); + let child_ref = *child.reference(); + + dag_state.insert_general_blocks( + vec![parent_a, parent_b, parent_c, child], + DataSource::BlockBundleStreaming, + ); + + assert!(!dag_state.apply_starfish_rbc_delivery_refs_for_test(&[child_ref])); + assert!(!dag_state.has_clean_vertex(&child_ref)); + assert!( + dag_state.apply_starfish_rbc_delivery_refs_for_test(&[parent_a_ref, parent_c_ref,]) + ); + assert!(!dag_state.has_clean_vertex(&child_ref)); + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&[parent_b_ref])); + assert!(dag_state.has_clean_vertex(&child_ref)); + } + + #[test] + fn starfish_rbc_delivery_before_dirty_insertion_is_latched() { + let dag_state = open_test_dag_state_for("starfish-rbc", 0); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let block = make_starfish_rbc_block(1, 1, genesis, Vec::new()); + let block_ref = *block.reference(); + + assert!( + !dag_state.apply_starfish_rbc_delivery_refs_for_test(&[block_ref]), + "delivery is retained but cannot activate an absent dirty carrier" + ); + assert!(!dag_state.has_clean_vertex(&block_ref)); + + dag_state.insert_general_block(block, DataSource::BlockBundleStreaming); + assert!( + dag_state.has_clean_vertex(&block_ref), + "later dirty insertion must consume the exact-reference delivery latch" + ); + } + + #[test] + fn starfish_rbc_disables_reference_inferred_cleanliness() { + let dag_state = open_test_dag_state_for("starfish-rbc", 0); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let target = make_starfish_rbc_block(1, 1, genesis.clone(), Vec::new()); + let filler_a = make_starfish_rbc_block(0, 1, genesis.clone(), Vec::new()); + let filler_b = make_starfish_rbc_block(2, 1, genesis, Vec::new()); + let target_ref = *target.reference(); + let round_one_refs = vec![target_ref, *filler_a.reference(), *filler_b.reference()]; + let supporter_a = make_starfish_rbc_block(0, 2, round_one_refs.clone(), Vec::new()); + let supporter_b = make_starfish_rbc_block(2, 2, round_one_refs, Vec::new()); + + dag_state.insert_general_blocks( + vec![target, filler_a, filler_b, supporter_a, supporter_b], + DataSource::BlockBundleStreaming, + ); + + assert!( + !dag_state.has_clean_vertex(&target_ref), + "f+1 dirty descendants are not a Starfish-RBC delivery certificate" + ); + } + + #[test] + fn starfish_rbc_rejects_bls_verification_as_a_clean_capability() { + let dag_state = open_test_dag_state_for("starfish-rbc", 0); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let block = make_starfish_rbc_block(1, 1, genesis, Vec::new()); + let block_ref = *block.reference(); + + // Neither event ordering may bypass the typed RBC delivery boundary. + assert!( + !dag_state.apply_certificate_events(vec![CertificateEvent::BlockVerified(block_ref,)]) + ); + dag_state.insert_general_block(block, DataSource::BlockBundleStreaming); + assert!(!dag_state.has_clean_vertex(&block_ref)); + assert!( + !dag_state.apply_certificate_events(vec![CertificateEvent::BlockVerified(block_ref,)]) + ); + assert!(!dag_state.has_clean_vertex(&block_ref)); + + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&[block_ref])); + assert!(dag_state.has_clean_vertex(&block_ref)); + } + + #[test] + fn starfish_rbc_dirty_quorum_cannot_advance_proposal_readiness() { + let dag_state = open_test_dag_state_for("starfish-rbc", 0); + let committee = Committee::new_for_benchmarks(4); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let round_one: Vec<_> = (0..3) + .map(|authority| make_starfish_rbc_block(authority, 1, genesis.clone(), Vec::new())) + .collect(); + let round_one_refs: Vec<_> = round_one.iter().map(|block| *block.reference()).collect(); + + dag_state.insert_general_blocks(round_one, DataSource::BlockBundleStreaming); + assert_eq!(dag_state.threshold_clock_round(), 2); + assert_eq!(dag_state.proposal_round(), 1); + assert!(!dag_state.is_ready_for_new_block( + 2, + &[committee.elect_leader(1)], + false, + 0, + committee.as_ref(), + )); + + let version_before_delivery = dag_state.round_version(1); + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&round_one_refs)); + assert_eq!(dag_state.proposal_round(), 2); + assert!(dag_state.is_ready_for_new_block( + 2, + &[committee.elect_leader(1)], + false, + 0, + committee.as_ref(), + )); + assert!( + dag_state.round_version(1) > version_before_delivery, + "clean activation must invalidate consensus round caches" + ); + } + + #[test] + fn starfish_rbc_acknowledgment_waits_for_both_cleanliness_and_data() { + let dag_state = open_test_dag_state_for("starfish-rbc", 0); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let transactions = vec![BaseTransaction::Share(Transaction::new(vec![1, 2, 3]))]; + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded = encoder.encode_transactions(&transactions, 2, 2); + + // Data first: insertion marks availability, but the dirty header must + // not enter the pending acknowledgment queue. + let mut data_first = VerifiedBlock::new_starfish_rbc( + 1, + 1, + genesis.clone(), + Vec::new(), + 1, + transactions.clone(), + Some(encoded.clone()), + ); + data_first.preserialize(); + let data_first = Data::new(data_first); + let data_first_ref = *data_first.reference(); + dag_state.insert_general_block(data_first, DataSource::BlockBundleStreaming); + assert!(dag_state.is_data_available(&data_first_ref)); + assert!(dag_state.get_pending_acknowledgment(1).is_empty()); + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&[data_first_ref])); + assert_eq!( + dag_state.get_pending_acknowledgment(1), + vec![data_first_ref] + ); + + // Clean first: the header carries the non-empty commitment but no + // local payload. Attaching the verified payload later queues it once. + let mut clean_first = VerifiedBlock::new_starfish_rbc( + 2, + 1, + genesis, + Vec::new(), + 2, + Vec::new(), + Some(encoded.clone()), + ); + clean_first.preserialize(); + let clean_first = Data::new(clean_first); + let clean_first_ref = *clean_first.reference(); + dag_state.insert_general_block(clean_first, DataSource::BlockBundleStreaming); + assert!(dag_state.apply_starfish_rbc_delivery_refs_for_test(&[clean_first_ref])); + assert!(dag_state.get_pending_acknowledgment(1).is_empty()); + + let mut transaction_data = TransactionData::new(transactions); + transaction_data.preserialize(); + let (commitment, proof) = + TransactionsCommitment::new_from_encoded_transactions(&encoded, 2); + let mut shard = ProvableShard::new(encoded[2].clone(), 2, proof, commitment); + shard.preserialize(); + assert!(dag_state.attach_transaction_data( + clean_first_ref, + &transaction_data, + &shard, + DataSource::ShardReconstructor, + )); + assert_eq!( + dag_state.get_pending_acknowledgment(1), + vec![clean_first_ref] + ); + } + #[test] fn sailfish_pending_certified_refs_are_buffered_until_flushed() { let dag_state = open_test_dag_state_for("sailfish-pp", 0); @@ -4964,6 +5365,12 @@ mod tests { ConsensusProtocol::Starfish.default_dissemination_mode(), DisseminationMode::PushUseful ); + assert_eq!( + ConsensusProtocol::StarfishRbc.default_dissemination_mode(), + DisseminationMode::PushUseful + ); + assert!(ConsensusProtocol::StarfishRbc.is_starfish_rbc()); + assert!(ConsensusProtocol::StarfishRbc.uses_dual_dag()); assert_eq!( ConsensusProtocol::StarfishSpeed.default_dissemination_mode(), DisseminationMode::PushUseful @@ -4985,6 +5392,7 @@ mod tests { ("mysticeti", ConsensusProtocol::Mysticeti), ("cordial-miners", ConsensusProtocol::CordialMiners), ("starfish", ConsensusProtocol::Starfish), + ("starfish-rbc", ConsensusProtocol::StarfishRbc), ("starfish-speed", ConsensusProtocol::StarfishSpeed), ("starfish-bls", ConsensusProtocol::StarfishBls), ("sailfish-pp", ConsensusProtocol::SailfishPlusPlus), @@ -5014,6 +5422,16 @@ mod tests { } } + assert_eq!( + ProtocolConfig::from_selection("starfish-rbc", Some("mac")).unwrap(), + ProtocolConfig { + consensus_protocol: ConsensusProtocol::StarfishRbc, + block_authentication_scheme: BlockAuthenticationScheme::MacVector, + } + ); + assert!(ProtocolConfig::from_selection("starfish", Some("mac")).is_err()); + assert!(ProtocolConfig::from_str("starfish-rbc-mac").is_err()); + for (name, consensus_protocol) in [ ("starfish-mac", ConsensusProtocol::Starfish), ("starfish-speed-mac", ConsensusProtocol::StarfishSpeed), diff --git a/crates/starfish-core/src/lib.rs b/crates/starfish-core/src/lib.rs index 4c72bbe7..bd6068cc 100644 --- a/crates/starfish-core/src/lib.rs +++ b/crates/starfish-core/src/lib.rs @@ -29,7 +29,8 @@ pub mod prometheus; mod rocks_store; mod runtime; pub mod shard_reconstructor; -mod starfish_rbc; +pub mod starfish_rbc; +mod starfish_rbc_service; mod stat; mod state; pub(crate) mod store; diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 77bf4bf7..b1de986e 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -132,6 +132,8 @@ pub struct Metrics { // per-request-type network message counters pub network_requests_sent_total: IntCounterVec, pub network_requests_received_total: IntCounterVec, + pub network_message_bytes_sent_total: IntCounterVec, + pub network_message_bytes_received_total: IntCounterVec, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -500,6 +502,20 @@ impl Metrics { registry, ) .unwrap(), + network_message_bytes_sent_total: register_int_counter_vec_with_registry!( + "network_message_bytes_sent_total", + "Total framed network-message bytes sent, by type", + &["request_type"], + registry, + ) + .unwrap(), + network_message_bytes_received_total: register_int_counter_vec_with_registry!( + "network_message_bytes_received_total", + "Total framed network-message bytes received, by type", + &["request_type"], + registry, + ) + .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index ec9942eb..829df318 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -44,6 +44,10 @@ use crate::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, + starfish_rbc::RbcProtocolInstanceId, + starfish_rbc_service::{ + RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, + }, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, @@ -55,6 +59,7 @@ use crate::{ const MAX_FILTER_SIZE: usize = 100_000; const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; +const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); /// Enforce the MAC experiment's transport contract before cryptographic /// verification: @@ -99,15 +104,20 @@ fn verify_mac_transport( } /// Prepare blocks forwarded through relay or synchronization paths for a -/// specific peer. MAC-authenticated blocks retain their complete vector only -/// at direct recipients; forwarding selects the destination's tag. A +/// specific peer. Legacy MAC-experiment blocks retain their complete vector +/// only at direct recipients; forwarding selects the destination's tag. A /// tag-only copy cannot be forwarded again and is therefore omitted. +/// Starfish-RBC carriers are authentication-free and remain forwardable: the +/// separate RBC service, rather than the carrier, controls clean admission. pub(crate) fn prepare_forwarded_blocks_for_peer( authentication_scheme: BlockAuthenticationScheme, + consensus_protocol: ConsensusProtocol, recipient: AuthorityIndex, blocks: Vec>, ) -> Vec> { - if authentication_scheme != BlockAuthenticationScheme::MacVector { + if authentication_scheme != BlockAuthenticationScheme::MacVector + || consensus_protocol.is_starfish_rbc() + { return blocks; } @@ -691,6 +701,7 @@ struct ConnectionHandler header_tx: mpsc::UnboundedSender<(Vec>, DataSource)>, bls_service: Option, sailfish_service: Option, + starfish_rbc_service: Option, } impl ConnectionHandler { @@ -732,6 +743,7 @@ impl ConnectionHandler ConnectionHandler ConnectionHandler ConnectionHandler { + if let Some(ref rbc) = self.starfish_rbc_service { + if let Err(error) = rbc.direct_initial(self.peer_id, proposal) { + tracing::warn!("Failed to forward Starfish-RBC INIT: {error}"); + } + } + } + NetworkMessage::RbcPhase(message) => { + if let Some(ref rbc) = self.starfish_rbc_service { + if let Err(error) = rbc.phase(self.peer_id, message) { + tracing::warn!("Failed to forward Starfish-RBC phase: {error}"); + } + } + } + NetworkMessage::RbcHeaderRequest(block_ref) => { + if let Some(ref rbc) = self.starfish_rbc_service { + if let Err(error) = rbc.header_request(self.peer_id, block_ref) { + tracing::warn!("Failed to forward Starfish-RBC header request: {error}"); + } + } + } + NetworkMessage::RbcHeaderResponse(header) => { + if let Some(ref rbc) = self.starfish_rbc_service { + if let Err(error) = rbc.header_response(self.peer_id, header) { + tracing::warn!("Failed to forward Starfish-RBC header response: {error}"); + } + } + } } true } @@ -1014,6 +1056,7 @@ impl ConnectionHandler { @@ -1038,6 +1081,7 @@ impl ConnectionHandler ConnectionHandler ConnectionHandler ConnectionHandler ConnectionHandler { bls_event_task: Option>, bls_broadcast_task: Option>, sf_event_task: Option>, + rbc_event_task: Option>, + rbc_service_task: Option>, cordial_knowledge_task: JoinHandle<()>, } @@ -1501,11 +1551,19 @@ pub struct NetworkSyncerInner { pub cordial_knowledge: CordialKnowledgeHandle, /// Per-peer message senders for direct unicast (e.g. DAC partial sigs). pub peer_senders: parking_lot::RwLock>>, + /// Nonblocking ingress to per-connection RBC outbound workers. Keeping + /// these queues separate prevents one backpressured peer from delaying + /// another peer or the actor's local HeaderStaged/Delivered effects. + rbc_peer_senders: + parking_lot::RwLock>>, pub leader_timeout: Duration, pub soft_block_timeout: Duration, /// Sailfish++ service handle for sending control messages /// (timeout/no-vote). None for non-SailfishPlusPlus protocols. pub sailfish_handle: Option, + /// Central Starfish-RBC service. Connection workers only forward their + /// trusted peer identity and wire payload into this single owner. + pub(crate) starfish_rbc_service: Option, /// Wall-clock at NetworkSyncer start; consumed by time-dependent /// Byzantine strategies (e.g. RampUpWithholding) to ramp behavior /// over a fixed schedule. @@ -1513,7 +1571,7 @@ pub struct NetworkSyncerInner { } impl NetworkSyncer { - pub fn start( + pub async fn start( network: Network, mut core: Core, mut commit_observer: C, @@ -1560,7 +1618,42 @@ impl NetworkSyncer let sf_handle_for_inner = sf_msg_tx .as_ref() .map(|tx| SailfishServiceHandle::new(tx.clone())); - let mut syncer = Syncer::new( + let (starfish_rbc_service, rbc_event_rx, rbc_service_task) = + if dag_state.consensus_protocol.is_starfish_rbc() { + let protocol_instance = node_parameters + .starfish_rbc_protocol_instance + .and_then(|bytes| RbcProtocolInstanceId::new(bytes).ok()) + .expect( + "validated Starfish-RBC configuration must contain a nonzero protocol instance", + ); + let initial_authenticator = match dag_state.block_authentication_scheme { + BlockAuthenticationScheme::Ed25519 => { + RbcInitialAuthenticator::Ed25519(core.get_signer().clone()) + } + BlockAuthenticationScheme::MlDsa44 => { + RbcInitialAuthenticator::MlDsa44(core.get_ml_dsa_44_signer().clone()) + } + BlockAuthenticationScheme::MlDsa65 => { + RbcInitialAuthenticator::MlDsa65(core.get_ml_dsa_65_signer().clone()) + } + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, + }; + let (service, events, task) = start_starfish_rbc_service( + committee.clone(), + dag_state.get_own_authority_index(), + protocol_instance, + dag_state.block_authentication_scheme, + mac_keys.clone(), + initial_authenticator, + dag_state.highest_round(), + STARFISH_RBC_HEADER_RETRY_INTERVAL, + ) + .expect("validated Starfish-RBC configuration must start its service"); + (Some(service), Some(events), Some(task)) + } else { + (None, None, None) + }; + let syncer = Syncer::new( core, NetworkSyncSignals { block_ready_notify: block_ready_notify.clone(), @@ -1570,10 +1663,14 @@ impl NetworkSyncer metrics.clone(), bls_msg_tx.clone(), sf_msg_tx.clone(), + starfish_rbc_service.clone(), ); let initial_round = syncer.core().next_block_round(); - syncer.force_new_block(initial_round); let syncer = CoreThreadDispatcher::start(syncer); + // Await the initial command while the async RBC actor remains + // schedulable. The command itself runs on the dedicated core thread, + // where synchronous local-INIT selection is safe. + syncer.force_new_block(initial_round).await; let (stop_sender, stop_receiver) = mpsc::channel(1); // Occupy the only available permit, so that all other // calls to send() will block. @@ -1585,6 +1682,7 @@ impl NetworkSyncer | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::SparseStarfishSpeed + | ConsensusProtocol::StarfishRbc ); let gc_round = Arc::new(AtomicU32::new(dag_state.gc_round())); let (shard_tx, decoded_rx) = if is_starfish { @@ -1630,12 +1728,85 @@ impl NetworkSyncer shard_tx: parking_lot::Mutex::new(shard_tx), cordial_knowledge: cordial_knowledge_handle, peer_senders: parking_lot::RwLock::new(AHashMap::new()), + rbc_peer_senders: parking_lot::RwLock::new(AHashMap::new()), leader_timeout: node_parameters.leader_timeout, soft_block_timeout: node_parameters.soft_block_timeout, sailfish_handle: sf_handle_for_inner, + starfish_rbc_service: starfish_rbc_service.clone(), start_time: std::time::Instant::now(), }); + // Bridge the single-owner RBC actor to direct network unicasts and to + // the core thread's dirty/clean DAG boundaries. Header staging never + // implies delivery; only a typed `Delivered` effect can mark a vertex + // clean. + let rbc_event_task = rbc_event_rx.map(|mut event_rx| { + let event_inner = inner.clone(); + handle.spawn(async move { + while let Some(event) = event_rx.recv().await { + match event { + RbcServiceEvent::Network { recipient, message } => { + let sender = + event_inner.rbc_peer_senders.read().get(&recipient).cloned(); + if let Some(sender) = sender { + if sender.send(message).is_err() { + tracing::debug!( + "Starfish-RBC outbound worker for authority {} stopped", + recipient + ); + } + } else { + // Local INITs and phase intents are retained by + // the actor and replayed when this peer connects. + tracing::debug!( + "Deferring Starfish-RBC message for disconnected authority {}", + recipient + ); + } + } + RbcServiceEvent::HeaderStaged(header) => { + let mut block = header.header().to_authentication_free_block(); + block.preserialize(); + let block_ref = *block.reference(); + event_inner + .cordial_knowledge + .send(CordialKnowledgeMessage::DagParts { + headers: vec![block_ref], + shards: Vec::new(), + }); + let (missing_parents, _) = event_inner + .syncer + .add_headers( + vec![Data::new(block)], + DataSource::BlockBundleStreamingHeader, + ) + .await; + if !missing_parents.is_empty() { + tracing::debug!( + "Starfish-RBC staged header {} waits for dependencies {:?}", + block_ref, + missing_parents + ); + } + } + RbcServiceEvent::Delivered(header) => { + event_inner + .syncer + .apply_starfish_rbc_deliveries(vec![header]) + .await; + } + RbcServiceEvent::Rejected { peer, error } => { + tracing::warn!( + "Rejected Starfish-RBC input from {:?}: {}", + peer, + error + ); + } + } + } + }) + }); + // Start bridge task that forwards reconstructed transaction data to core let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); @@ -1919,6 +2090,8 @@ impl NetworkSyncer bls_event_task, bls_broadcast_task, sf_event_task, + rbc_event_task, + rbc_service_task, cordial_knowledge_task, } } @@ -1956,6 +2129,14 @@ impl NetworkSyncer sf_task.abort(); sf_task.await.ok(); } + // Stop RBC event ingress first, but keep the service actor alive while + // the core queue drains: an already queued core action may still + // synchronously select another local INIT. + if let Some(rbc_task) = self.rbc_event_task { + rbc_task.abort(); + rbc_task.await.ok(); + } + let rbc_service_task = self.rbc_service_task; // Stop the cordial knowledge actor. self.cordial_knowledge_task.abort(); self.cordial_knowledge_task.await.ok(); @@ -1981,7 +2162,16 @@ impl NetworkSyncer } } }; - inner.syncer.stop() + // `inner` is now exclusive, so no auxiliary task can enqueue after + // this FIFO barrier. Awaiting it keeps the runtime available to the + // RBC actor while any earlier core action completes. + let _ = inner.syncer.missing_parent_references().await; + let syncer = inner.syncer.stop(); + if let Some(rbc_service_task) = rbc_service_task { + rbc_service_task.abort(); + rbc_service_task.await.ok(); + } + syncer } async fn run( @@ -2112,6 +2302,25 @@ impl NetworkSyncer .peer_senders .write() .insert(peer_id, connection.sender.clone()); + let rbc_outbound_task = inner.starfish_rbc_service.as_ref().map(|_| { + let (rbc_sender, mut rbc_receiver) = mpsc::unbounded_channel(); + inner.rbc_peer_senders.write().insert(peer_id, rbc_sender); + let network_sender = connection.sender.clone(); + Handle::current().spawn(async move { + while let Some(message) = rbc_receiver.recv().await { + send_network_message_reliably(&network_sender, message).await; + } + }) + }); + if let Some(ref rbc) = inner.starfish_rbc_service { + if let Err(error) = rbc.peer_connected(peer_id) { + tracing::warn!( + "Failed to notify Starfish-RBC service that authority {} connected: {}", + peer_id, + error + ); + } + } if inner.dag_state.consensus_protocol.uses_bls() { for (round, signature) in inner.dag_state.precomputed_round_sigs() { @@ -2150,7 +2359,21 @@ impl NetworkSyncer } tracing::debug!("Connection between {own_id} and {peer_id} is dropped"); + if let Some(ref rbc) = inner.starfish_rbc_service { + if let Err(error) = rbc.peer_disconnected(peer_id) { + tracing::warn!( + "Failed to notify Starfish-RBC service that authority {} disconnected: {}", + peer_id, + error + ); + } + } inner.peer_senders.write().remove(&peer_id); + inner.rbc_peer_senders.write().remove(&peer_id); + if let Some(rbc_outbound_task) = rbc_outbound_task { + rbc_outbound_task.abort(); + rbc_outbound_task.await.ok(); + } inner.syncer.authority_connection(peer_id, false).await; handler.shutdown().await; block_fetcher.remove_authority(peer_id).await; @@ -2660,6 +2883,7 @@ mod tests { let round_gap_blocks = prepare_forwarded_blocks_for_peer( BlockAuthenticationScheme::MacVector, + ConsensusProtocol::Bluestreak, 0, vec![Data::new(full)], ); diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index e62c4426..5ee30976 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -28,6 +28,7 @@ use crate::{ metrics::{Metrics, print_network_address_table}, runtime, runtime::JoinHandle, + starfish_rbc::{RbcCanonicalHeader, RbcHeaderProposal, RbcPhase, RbcPhaseMessage}, stat::HistogramSender, types::{ AuthorityIndex, AuthoritySet, BlockReference, CertMessage, CertMessageKind, PartialSig, @@ -178,6 +179,17 @@ pub enum NetworkMessage { round: RoundNumber, known_authorities: AuthoritySet, }, + /// Starfish-RBC: direct-author canonical header and receiver-specific + /// initial proof. + RbcInitial(RbcHeaderProposal), + /// Starfish-RBC: direct, recipient-specific ECHO or READY testimony. + RbcPhase(RbcPhaseMessage), + /// Starfish-RBC: request canonical header content for a phase-evidenced + /// block reference. + RbcHeaderRequest(BlockReference), + /// Starfish-RBC: return canonical header content. The receiver recomputes + /// and checks its content-addressed reference before accepting it. + RbcHeaderResponse(RbcCanonicalHeader), } impl NetworkMessage { @@ -198,6 +210,13 @@ impl NetworkMessage { Self::SailfishNoVote(_) => "sailfish_no_vote", Self::UnprovableCertificateRequest { .. } => "unprovable_cert_request", Self::RoundGapRequest { .. } => "round_gap_request", + Self::RbcInitial(_) => "rbc_initial", + Self::RbcPhase(message) => match message.phase() { + RbcPhase::Echo => "rbc_echo", + RbcPhase::Ready => "rbc_ready", + }, + Self::RbcHeaderRequest(_) => "rbc_header_request", + Self::RbcHeaderResponse(_) => "rbc_header_response", } } } @@ -520,6 +539,7 @@ impl Worker { let start = Instant::now(); let bytes_sent_total = metrics.bytes_sent_total.clone(); let network_requests_sent_total = metrics.network_requests_sent_total.clone(); + let network_message_bytes_sent_total = metrics.network_message_bytes_sent_total.clone(); // Spawn the first task for handling pings let writer_clone = Arc::clone(&writer); @@ -617,12 +637,13 @@ impl Worker { } else { serialized }; + let framed_len = wire_bytes.len() as u64 + 4; match async { let mut writer_guard = writer.lock().await; writer_guard.write_u32(wire_bytes.len() as u32).await?; - bytes_sent_total.inc_by(wire_bytes.len() as u64 + 4); + bytes_sent_total.inc_by(framed_len); writer_guard.write_all(&wire_bytes).await } .await @@ -631,6 +652,9 @@ impl Worker { network_requests_sent_total .with_label_values(&[request_type]) .inc(); + network_message_bytes_sent_total + .with_label_values(&[request_type]) + .inc_by(framed_len); } Err(e) => { tracing::error!("Failed to write message: {e}"); @@ -661,6 +685,7 @@ impl Worker { let writer = writer.clone(); let bytes_sent_total = bytes_sent_total.clone(); let network_requests_sent_total = network_requests_sent_total.clone(); + let network_message_bytes_sent_total = network_message_bytes_sent_total.clone(); let bytes_uncompressed_sent_total = metrics.bytes_uncompressed_sent_total.clone(); let latency = generate_latency(effective_latency(connection_latency, connection_scaled)); @@ -674,13 +699,14 @@ impl Worker { } else { serialized }; + let framed_len = wire_bytes.len() as u64 + 4; tokio::time::sleep(latency).await; match async { let mut writer_guard = writer.lock().await; writer_guard.write_u32(wire_bytes.len() as u32).await?; - bytes_sent_total.inc_by(wire_bytes.len() as u64 + 4); + bytes_sent_total.inc_by(framed_len); writer_guard.write_all(&wire_bytes).await } .await @@ -689,6 +715,9 @@ impl Worker { network_requests_sent_total .with_label_values(&[request_type]) .inc(); + network_message_bytes_sent_total + .with_label_values(&[request_type]) + .inc_by(framed_len); } Err(e) => { tracing::error!("Failed to write message: {e}"); @@ -766,6 +795,10 @@ impl Worker { match deserialize_result { Some(message) => { let request_type = message.request_type(); + metrics + .network_message_bytes_received_total + .with_label_values(&[request_type]) + .inc_by(read as u64 + 4); if sender.send(message).await.is_err() { // todo - pass signal to break main loop return Ok(()); @@ -926,3 +959,66 @@ fn decode_ping(message: &[u8]) -> i64 { m.copy_from_slice(message); // asserts message.len() == 8 i64::from_le_bytes(m) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + crypto::{MacTag, TransactionsCommitment, dummy_signer}, + starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, + }; + + fn variant_index(message: &NetworkMessage) -> u32 { + let bytes = bincode::serialize(message).unwrap(); + u32::from_le_bytes(bytes[..4].try_into().unwrap()) + } + + #[test] + fn rbc_wire_variants_are_append_only_and_roundtrip() { + // This pre-existing last variant is frozen at index 10. Adding RBC + // messages must not renumber any legacy bincode discriminant. + let legacy = NetworkMessage::RoundGapRequest { + round: 7, + known_authorities: AuthoritySet::default(), + }; + assert_eq!(variant_index(&legacy), 10); + + let header = RbcCanonicalHeader::try_new( + 0, + 1, + Vec::new(), + Vec::new(), + 11, + TransactionsCommitment::default(), + ) + .unwrap(); + let block_ref = header.reference(); + let initial = NetworkMessage::RbcInitial(RbcHeaderProposal::new( + header.clone(), + RbcInitialProof::Ed25519(dummy_signer().sign_digest(&[0xA1; 32])), + )); + let phase = NetworkMessage::RbcPhase(RbcPhaseMessage::new_for_test( + block_ref, + 1, + 2, + RbcPhase::Ready, + MacTag::from_bytes([0xA2; 32]), + )); + let request = NetworkMessage::RbcHeaderRequest(block_ref); + let response = NetworkMessage::RbcHeaderResponse(header); + + for (message, expected_index, expected_kind) in [ + (initial, 11, "rbc_initial"), + (phase, 12, "rbc_ready"), + (request, 13, "rbc_header_request"), + (response, 14, "rbc_header_response"), + ] { + assert_eq!(variant_index(&message), expected_index); + assert_eq!(message.request_type(), expected_kind); + let encoded = bincode::serialize(&message).unwrap(); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + assert_eq!(decoded.request_type(), expected_kind); + assert_eq!(variant_index(&decoded), expected_index); + } + } +} diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index 261bdf45..6befce63 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -3,11 +3,9 @@ //! Synchronous reliable-broadcast kernel for Starfish-RBC. //! -//! This module is deliberately not connected to networking or DAG admission -//! yet. The next milestones will supply content-validated headers and expand -//! multicast effects into recipient-specific network messages. - -#![allow(dead_code)] +//! Networking and DAG admission remain outside the kernel: the service adapter +//! supplies content-validated headers and expands typed multicast effects into +//! recipient-specific messages. use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; @@ -23,7 +21,7 @@ use crate::{ types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - TimestampNs, compress_acknowledgments, expand_acknowledgments, + TimestampNs, VerifiedBlock, compress_acknowledgments, expand_acknowledgments, }, }; @@ -151,7 +149,7 @@ impl RbcAckFields { /// Acknowledgments stay canonically compressed on wire, but the digest hashes /// their expanded logical vector with an explicit boundary from parents. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub(crate) struct RbcCanonicalHeader { +pub struct RbcCanonicalHeader { reference: BlockReference, #[serde(with = "bounded_references")] block_references: Vec, @@ -161,7 +159,7 @@ pub(crate) struct RbcCanonicalHeader { } impl RbcCanonicalHeader { - fn try_new( + pub(crate) fn try_new( authority: AuthorityIndex, round: RoundNumber, block_references: Vec, @@ -289,15 +287,15 @@ impl RbcCanonicalHeader { }) } - pub(crate) fn reference(&self) -> BlockReference { + pub fn reference(&self) -> BlockReference { self.reference } - pub(crate) fn block_references(&self) -> &[BlockReference] { + pub fn block_references(&self) -> &[BlockReference] { &self.block_references } - pub(crate) fn acknowledgment_references(&self) -> Vec { + pub fn acknowledgment_references(&self) -> Vec { self.acknowledgments.logical(&self.block_references) } @@ -308,58 +306,36 @@ impl RbcCanonicalHeader { } } - pub(crate) fn meta_creation_time_ns(&self) -> TimestampNs { + pub fn meta_creation_time_ns(&self) -> TimestampNs { self.meta_creation_time_ns } - pub(crate) fn transactions_commitment(&self) -> TransactionsCommitment { + pub fn transactions_commitment(&self) -> TransactionsCommitment { self.transactions_commitment } - fn encoded_content_size(&self, acknowledgment_count: usize) -> Result { - let reference_count = self - .block_references - .len() - .checked_add(acknowledgment_count) - .ok_or(RbcError::HeaderContentTooLarge)?; - RBC_BLOCK_REFERENCE_SIZE - .checked_mul(reference_count) - .and_then(|size| size.checked_add(RBC_HEADER_FIXED_CONTENT_SIZE)) - .ok_or(RbcError::HeaderContentTooLarge) - } -} - -/// An intrinsically validated header retained by `Arc` for as long as the RBC -/// state may advertise this validator as a holder. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct PinnedRbcHeader { - header: Arc, - committee_id: RbcCommitteeId, -} - -impl PinnedRbcHeader { - fn validate_with_committee_id( - header: RbcCanonicalHeader, - committee: &Committee, - committee_id: RbcCommitteeId, - ) -> Result { - let block_ref = header.reference; + /// Validate canonical header content against an already validated static + /// committee without deriving the committee identifier. + /// + /// This is the shared structural boundary for RBC INIT/recovery and for a + /// later normal block-batch payload carrier. Global committee invariants + /// are checked once when the RBC context is created; this per-header path + /// touches only authorities referenced by the header. + pub(crate) fn validate_for_committee(&self, committee: &Committee) -> Result<(), RbcError> { + let block_ref = self.reference; if block_ref.round == 0 { return Err(RbcError::GenesisSlot); } if !committee.known_authority(block_ref.authority) { return Err(RbcError::UnknownAuthority(block_ref.authority)); } - if !header - .acknowledgments - .is_canonical(&header.block_references) - { + if !self.acknowledgments.is_canonical(&self.block_references) { return Err(RbcError::NonCanonicalAcknowledgments); } - let acknowledgments = header.acknowledgment_references(); + let acknowledgments = self.acknowledgment_references(); for (field, count) in [ - ("parent", header.block_references.len()), + ("parent", self.block_references.len()), ("acknowledgment", acknowledgments.len()), ] { if count > MAX_RBC_REFERENCES_PER_FIELD { @@ -370,13 +346,13 @@ impl PinnedRbcHeader { }); } } - if header.encoded_content_size(acknowledgments.len())? > MAX_RBC_HEADER_CONTENT_SIZE { + if self.encoded_content_size(acknowledgments.len())? > MAX_RBC_HEADER_CONTENT_SIZE { return Err(RbcError::HeaderContentTooLarge); } let mut parent_set = AHashSet::new(); let mut previous_round_parents = StakeAggregator::::new(); - for parent in &header.block_references { + for parent in &self.block_references { if !committee.known_authority(parent.authority) { return Err(RbcError::UnknownAuthority(parent.authority)); } @@ -410,10 +386,10 @@ impl PinnedRbcHeader { let expected_digest = BlockDigest::new_starfish_rbc_header( block_ref.authority, block_ref.round, - &header.block_references, + &self.block_references, &acknowledgments, - header.meta_creation_time_ns, - header.transactions_commitment, + self.meta_creation_time_ns, + self.transactions_commitment, ); if expected_digest != block_ref.digest { return Err(RbcError::HeaderDigestMismatch { @@ -421,6 +397,59 @@ impl PinnedRbcHeader { actual: block_ref.digest, }); } + Ok(()) + } + + /// Convert canonical content into the existing header-only carrier. RBC + /// authorization remains external, so the compatibility header contains + /// no signature or MAC sidecar. + pub(crate) fn to_authentication_free_block(&self) -> VerifiedBlock { + VerifiedBlock::from_parts( + BlockHeader { + reference: self.reference, + block_references: self.block_references.clone(), + meta_creation_time_ns: self.meta_creation_time_ns, + authentication: BlockAuthentication::None, + transactions_commitment: Some(self.transactions_commitment), + ack: Some(self.acknowledgment_fields()), + strong_vote: None, + bls: None, + sailfish: None, + unprovable_certificate: None, + serialized: None, + }, + None, + ) + } + + fn encoded_content_size(&self, acknowledgment_count: usize) -> Result { + let reference_count = self + .block_references + .len() + .checked_add(acknowledgment_count) + .ok_or(RbcError::HeaderContentTooLarge)?; + RBC_BLOCK_REFERENCE_SIZE + .checked_mul(reference_count) + .and_then(|size| size.checked_add(RBC_HEADER_FIXED_CONTENT_SIZE)) + .ok_or(RbcError::HeaderContentTooLarge) + } +} + +/// An intrinsically validated header retained by `Arc` for as long as the RBC +/// state may advertise this validator as a holder. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedRbcHeader { + header: Arc, + committee_id: RbcCommitteeId, +} + +impl PinnedRbcHeader { + fn validate_with_committee_id( + header: RbcCanonicalHeader, + committee: &Committee, + committee_id: RbcCommitteeId, + ) -> Result { + header.validate_for_committee(committee)?; Ok(Self { header: Arc::new(header), committee_id, @@ -451,14 +480,43 @@ impl PinnedRbcHeader { } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub(crate) enum RbcInitialProof { +pub enum RbcInitialProof { Ed25519(SignatureBytes), MlDsa44(MlDsa44SignatureBytes), MlDsa65(MlDsa65SignatureBytes), Mac(MacTag), } +/// Direct-author Starfish-RBC header proposal carried on the wire. +/// +/// The proof is a sidecar over the canonical header reference. It is not part +/// of the content-addressed header identity. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcHeaderProposal { + header: RbcCanonicalHeader, + proof: RbcInitialProof, +} + +impl RbcHeaderProposal { + pub(crate) fn new(header: RbcCanonicalHeader, proof: RbcInitialProof) -> Self { + Self { header, proof } + } + + pub fn header(&self) -> &RbcCanonicalHeader { + &self.header + } + + pub fn proof(&self) -> &RbcInitialProof { + &self.proof + } + + pub(crate) fn into_parts(self) -> (RbcCanonicalHeader, RbcInitialProof) { + (self.header, self.proof) + } +} + impl RbcInitialProof { + #[allow(dead_code)] pub(crate) fn from_block_authentication( authentication: &BlockAuthentication, ) -> Result { @@ -621,8 +679,8 @@ impl RbcContext { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub(crate) enum RbcPhase { +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub enum RbcPhase { Echo, Ready, } @@ -637,7 +695,7 @@ impl RbcPhase { } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub(crate) struct RbcPhaseMessage { +pub struct RbcPhaseMessage { block_ref: BlockReference, sender: AuthorityIndex, recipient: AuthorityIndex, @@ -646,19 +704,36 @@ pub(crate) struct RbcPhaseMessage { } impl RbcPhaseMessage { - pub(crate) fn block_ref(&self) -> BlockReference { + #[cfg(test)] + pub(crate) fn new_for_test( + block_ref: BlockReference, + sender: AuthorityIndex, + recipient: AuthorityIndex, + phase: RbcPhase, + tag: MacTag, + ) -> Self { + Self { + block_ref, + sender, + recipient, + phase, + tag, + } + } + + pub fn block_ref(&self) -> BlockReference { self.block_ref } - pub(crate) fn sender(&self) -> AuthorityIndex { + pub fn sender(&self) -> AuthorityIndex { self.sender } - pub(crate) fn recipient(&self) -> AuthorityIndex { + pub fn recipient(&self) -> AuthorityIndex { self.recipient } - pub(crate) fn phase(&self) -> RbcPhase { + pub fn phase(&self) -> RbcPhase { self.phase } } @@ -716,10 +791,12 @@ pub(crate) enum RbcError { current: RoundNumber, proposed: RoundNumber, }, + #[allow(dead_code)] RetainedRoundRegression { current: RoundNumber, proposed: RoundNumber, }, + #[allow(dead_code)] RetainedRoundAheadOfLocal { local: RoundNumber, proposed: RoundNumber, @@ -1077,6 +1154,7 @@ impl StarfishRbcKernel { }) } + #[allow(dead_code)] pub(crate) fn context(&self) -> RbcContext { self.context } @@ -1096,6 +1174,7 @@ impl StarfishRbcKernel { Ok(()) } + #[allow(dead_code)] pub(crate) fn minimum_new_slot_round(&self) -> RoundNumber { self.minimum_new_slot_round } @@ -1105,6 +1184,7 @@ impl StarfishRbcKernel { /// call: the integration layer may advance it only when its recovery model /// proves that no newly observed slot below `round` is still required. /// Existing slots remain active so late evidence can complete totality. + #[allow(dead_code)] pub(crate) fn close_new_slots_before(&mut self, round: RoundNumber) -> Result<(), RbcError> { if round < self.minimum_new_slot_round { return Err(RbcError::RetainedRoundRegression { @@ -1555,12 +1635,28 @@ impl StarfishRbcKernel { Ok(()) } + #[allow(dead_code)] pub(crate) fn header_holders(&self, block_ref: &BlockReference) -> AuthoritySet { self.candidate(block_ref) .map(CandidateState::holders) .unwrap_or_default() } + /// Return the retained, content-validated header for a candidate. + /// + /// The service uses this accessor to answer recovery requests. Returning + /// the pin (rather than a detached header clone) preserves the invariant + /// that an honest ECHO/READY sender keeps the advertised content alive. + pub(crate) fn pinned_header( + &self, + block_ref: BlockReference, + ) -> Result, RbcError> { + self.validate_block_ref(&block_ref)?; + Ok(self + .candidate(&block_ref) + .and_then(|candidate| candidate.header.clone())) + } + /// Recreate the current fetch effect for a durable retry timer. The first /// `NeedHeader` effect is only a wake-up; recovery must retry until the /// content-validated header becomes locally pinned. @@ -2275,6 +2371,16 @@ mod tests { let extracted_with = RbcCanonicalHeader::from_block_header(&with_authentication).unwrap(); assert_eq!(extracted_without, canonical); assert_eq!(extracted_with, canonical); + canonical.validate_for_committee(&committee).unwrap(); + + let carrier = canonical.to_authentication_free_block(); + assert_eq!(carrier.reference(), &canonical.reference()); + assert_eq!(carrier.authentication(), &BlockAuthentication::None); + assert!(!carrier.has_transaction_data()); + assert_eq!( + RbcCanonicalHeader::from_block_header(carrier.header()).unwrap(), + canonical + ); let pinned = PinnedRbcHeader::validate(canonical.clone(), &committee).unwrap(); let retained = pinned.clone(); diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs new file mode 100644 index 00000000..3962fb1b --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -0,0 +1,1225 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Single-owner async adapter around the Starfish-RBC kernel. +//! +//! The service deliberately owns header-recovery state independently of any +//! one network connection. Connection workers only attach their trusted peer +//! identity and forward messages into this actor; reconnects therefore cannot +//! discard a pending recovery attempt. + +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, + sync::Arc, + time::Duration, +}; + +use ahash::{AHashMap, AHashSet}; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinHandle, + time::MissedTickBehavior, +}; + +use crate::{ + committee::Committee, + crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + network::NetworkMessage, + starfish_rbc::{ + PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, + RbcInitialHeaderOutcome, RbcInitialProof, RbcLocalInitial, RbcPhase, RbcPhaseMessage, + RbcProtocolInstanceId, StarfishRbcKernel, + }, + types::{ + AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, + RoundNumber, TimestampNs, + }, +}; + +const HEADER_REQUEST_FANOUT: usize = 2; + +/// Authentication material used only for the author's initial RBC proposal. +/// ECHO and READY always use the pairwise MAC keyring owned by the kernel. +#[derive(Clone)] +pub(crate) enum RbcInitialAuthenticator { + Ed25519(Signer), + MlDsa44(MlDsa44Signer), + MlDsa65(MlDsa65Signer), + Mac, +} + +impl RbcInitialAuthenticator { + fn scheme(&self) -> BlockAuthenticationScheme { + match self { + Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, + Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, + Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, + Self::Mac => BlockAuthenticationScheme::MacVector, + } + } +} + +/// Authentication-free inputs for atomically starting one local RBC slot. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RbcLocalHeader { + pub round: RoundNumber, + pub block_references: Vec, + pub acknowledgment_references: Vec, + pub meta_creation_time_ns: TimestampNs, + pub transactions_commitment: TransactionsCommitment, +} + +impl RbcLocalHeader { + pub(crate) fn from_canonical(header: &RbcCanonicalHeader) -> Self { + Self { + round: header.reference().round, + block_references: header.block_references().to_vec(), + acknowledgment_references: header.acknowledgment_references(), + meta_creation_time_ns: header.meta_creation_time_ns(), + transactions_commitment: header.transactions_commitment(), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum RbcServiceError { + Kernel(RbcError), + InitialAuthenticatorSchemeMismatch { + configured: BlockAuthenticationScheme, + supplied: BlockAuthenticationScheme, + }, + LocalAuthenticatorKeyMismatch(AuthorityIndex), + ZeroHeaderRetryInterval, + UnexpectedHeaderResponse(BlockReference), + HeaderResponseFromNonHolder { + block_ref: BlockReference, + peer: AuthorityIndex, + }, + ServiceStopped, +} + +impl From for RbcServiceError { + fn from(error: RbcError) -> Self { + Self::Kernel(error) + } +} + +impl fmt::Display for RbcServiceError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Kernel(error) => error.fmt(formatter), + Self::InitialAuthenticatorSchemeMismatch { + configured, + supplied, + } => write!( + formatter, + "Starfish-RBC initial authenticator mismatch: configured {configured:?}, \ + supplied {supplied:?}" + ), + Self::LocalAuthenticatorKeyMismatch(authority) => write!( + formatter, + "Starfish-RBC initial authenticator key does not match authority {authority}" + ), + Self::ZeroHeaderRetryInterval => { + formatter.write_str("Starfish-RBC header retry interval must be nonzero") + } + Self::UnexpectedHeaderResponse(block_ref) => write!( + formatter, + "unexpected Starfish-RBC header response for {block_ref}" + ), + Self::HeaderResponseFromNonHolder { block_ref, peer } => write!( + formatter, + "Starfish-RBC header response for {block_ref} came from non-holder {peer}" + ), + Self::ServiceStopped => formatter.write_str("Starfish-RBC service stopped"), + } + } +} + +impl Error for RbcServiceError {} + +/// Events consumed by the network/core integration bridge. +#[derive(Debug)] +pub(crate) enum RbcServiceEvent { + Network { + recipient: AuthorityIndex, + message: NetworkMessage, + }, + HeaderStaged(PinnedRbcHeader), + Delivered(PinnedRbcHeader), + Rejected { + peer: Option, + error: RbcServiceError, + }, +} + +enum RbcServiceMessage { + StartLocal { + header: RbcLocalHeader, + reply: oneshot::Sender>, + }, + DirectInitial { + peer: AuthorityIndex, + proposal: RbcHeaderProposal, + }, + Phase { + peer: AuthorityIndex, + message: RbcPhaseMessage, + }, + HeaderRequest { + peer: AuthorityIndex, + block_ref: BlockReference, + }, + HeaderResponse { + peer: AuthorityIndex, + header: RbcCanonicalHeader, + }, + PeerConnected(AuthorityIndex), + PeerDisconnected(AuthorityIndex), + #[allow(dead_code)] + AdvanceLocalRound { + round: RoundNumber, + reply: oneshot::Sender>, + }, + #[allow(dead_code)] + RetryHeaders(oneshot::Sender<()>), +} + +#[derive(Clone)] +pub(crate) struct RbcServiceHandle { + sender: mpsc::UnboundedSender, +} + +impl RbcServiceHandle { + #[allow(dead_code)] + pub(crate) async fn start_local_header( + &self, + header: RbcLocalHeader, + ) -> Result { + let (reply, receiver) = oneshot::channel(); + self.send(RbcServiceMessage::StartLocal { header, reply })?; + receiver + .await + .map_err(|_| RbcServiceError::ServiceStopped)? + } + + /// Synchronous entry point for the dedicated core thread. The reply is + /// completed only after the kernel has selected/pinned the slot and all + /// INIT/phase events have been enqueued, so legacy dissemination cannot + /// race ahead of local RBC authorization. + pub(crate) fn start_local_header_blocking( + &self, + header: RbcLocalHeader, + ) -> Result { + let (reply, receiver) = oneshot::channel(); + self.send(RbcServiceMessage::StartLocal { header, reply })?; + receiver + .blocking_recv() + .map_err(|_| RbcServiceError::ServiceStopped)? + } + + pub(crate) fn direct_initial( + &self, + peer: AuthorityIndex, + proposal: RbcHeaderProposal, + ) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::DirectInitial { peer, proposal }) + } + + pub(crate) fn phase( + &self, + peer: AuthorityIndex, + message: RbcPhaseMessage, + ) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::Phase { peer, message }) + } + + pub(crate) fn header_request( + &self, + peer: AuthorityIndex, + block_ref: BlockReference, + ) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::HeaderRequest { peer, block_ref }) + } + + pub(crate) fn header_response( + &self, + peer: AuthorityIndex, + header: RbcCanonicalHeader, + ) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::HeaderResponse { peer, header }) + } + + pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::PeerConnected(peer)) + } + + pub(crate) fn peer_disconnected(&self, peer: AuthorityIndex) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::PeerDisconnected(peer)) + } + + #[allow(dead_code)] + pub(crate) async fn advance_local_round( + &self, + round: RoundNumber, + ) -> Result<(), RbcServiceError> { + let (reply, receiver) = oneshot::channel(); + self.send(RbcServiceMessage::AdvanceLocalRound { round, reply })?; + receiver + .await + .map_err(|_| RbcServiceError::ServiceStopped)? + } + + /// Trigger a recovery wave immediately. Production also has an internal + /// periodic timer; this method is useful after topology changes and makes + /// retry behavior deterministic in tests. + #[allow(dead_code)] + pub(crate) async fn retry_headers(&self) -> Result<(), RbcServiceError> { + let (reply, receiver) = oneshot::channel(); + self.send(RbcServiceMessage::RetryHeaders(reply))?; + receiver.await.map_err(|_| RbcServiceError::ServiceStopped) + } + + fn send(&self, message: RbcServiceMessage) -> Result<(), RbcServiceError> { + self.sender + .send(message) + .map_err(|_| RbcServiceError::ServiceStopped) + } +} + +/// Start the single Starfish-RBC state-machine owner. +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_starfish_rbc_service( + committee: Arc, + own_authority: AuthorityIndex, + protocol_instance: RbcProtocolInstanceId, + initial_authentication: BlockAuthenticationScheme, + mac_keys: Arc>, + initial_authenticator: RbcInitialAuthenticator, + local_round: RoundNumber, + header_retry_interval: Duration, +) -> Result< + ( + RbcServiceHandle, + mpsc::UnboundedReceiver, + JoinHandle<()>, + ), + RbcServiceError, +> { + if header_retry_interval.is_zero() { + return Err(RbcServiceError::ZeroHeaderRetryInterval); + } + if initial_authenticator.scheme() != initial_authentication { + return Err(RbcServiceError::InitialAuthenticatorSchemeMismatch { + configured: initial_authentication, + supplied: initial_authenticator.scheme(), + }); + } + validate_local_authenticator(&committee, own_authority, &initial_authenticator)?; + + let kernel = StarfishRbcKernel::new( + committee.clone(), + own_authority, + protocol_instance, + initial_authentication, + mac_keys, + local_round, + )?; + let (message_tx, message_rx) = mpsc::unbounded_channel(); + let (event_tx, event_rx) = mpsc::unbounded_channel(); + let state = RbcServiceState { + committee, + own_authority, + initial_authenticator, + kernel, + events: event_tx, + connected_peers: AuthoritySet::default(), + pending_fetches: AHashMap::new(), + staged_notifications: AHashSet::new(), + retained_initials: BTreeMap::new(), + retained_phases: BTreeSet::new(), + }; + let task = tokio::spawn(run_service(state, message_rx, header_retry_interval)); + Ok((RbcServiceHandle { sender: message_tx }, event_rx, task)) +} + +fn validate_local_authenticator( + committee: &Committee, + own_authority: AuthorityIndex, + authenticator: &RbcInitialAuthenticator, +) -> Result<(), RbcServiceError> { + let matches = match authenticator { + RbcInitialAuthenticator::Ed25519(signer) => committee + .get_public_key(own_authority) + .is_some_and(|public_key| public_key == &signer.public_key()), + RbcInitialAuthenticator::MlDsa44(signer) => committee + .get_ml_dsa_44_public_key(own_authority) + .is_some_and(|public_key| public_key == &signer.public_key()), + RbcInitialAuthenticator::MlDsa65(signer) => committee + .get_ml_dsa_65_public_key(own_authority) + .is_some_and(|public_key| public_key == &signer.public_key()), + RbcInitialAuthenticator::Mac => committee.known_authority(own_authority), + }; + if matches { + Ok(()) + } else { + Err(RbcServiceError::LocalAuthenticatorKeyMismatch( + own_authority, + )) + } +} + +async fn run_service( + mut state: RbcServiceState, + mut messages: mpsc::UnboundedReceiver, + header_retry_interval: Duration, +) { + let mut retry = tokio::time::interval(header_retry_interval); + retry.set_missed_tick_behavior(MissedTickBehavior::Skip); + retry.tick().await; + + loop { + tokio::select! { + maybe_message = messages.recv() => { + let Some(message) = maybe_message else { + break; + }; + state.process_message(message); + } + _ = retry.tick() => state.retry_pending_headers(), + } + } +} + +struct PendingHeaderFetch { + holders: AuthoritySet, + requested_from: AuthoritySet, +} + +struct RbcServiceState { + committee: Arc, + own_authority: AuthorityIndex, + initial_authenticator: RbcInitialAuthenticator, + kernel: StarfishRbcKernel, + events: mpsc::UnboundedSender, + connected_peers: AuthoritySet, + pending_fetches: AHashMap, + staged_notifications: AHashSet, + /// Recipient-specialized local proposals retained for replay after a + /// connection is replaced. Version one keeps these for the run. + retained_initials: BTreeMap<(BlockReference, AuthorityIndex), RbcHeaderProposal>, + /// Authorized local phase intents. Tags are rematerialized for the peer + /// on replay rather than retaining or cloning a tagged wire message. + retained_phases: BTreeSet<(BlockReference, RbcPhase)>, +} + +impl RbcServiceState { + fn process_message(&mut self, message: RbcServiceMessage) { + match message { + RbcServiceMessage::StartLocal { header, reply } => { + let result = self.start_local_header(header); + let _ = reply.send(result); + } + RbcServiceMessage::DirectInitial { peer, proposal } => { + self.accept_direct_initial(peer, proposal); + } + RbcServiceMessage::Phase { peer, message } => { + match self.kernel.handle_phase(peer, message) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(peer), error.into()), + } + } + RbcServiceMessage::HeaderRequest { peer, block_ref } => { + self.answer_header_request(peer, block_ref); + } + RbcServiceMessage::HeaderResponse { peer, header } => { + self.accept_header_response(peer, header); + } + RbcServiceMessage::PeerConnected(peer) => self.peer_connected(peer), + RbcServiceMessage::PeerDisconnected(peer) => self.peer_disconnected(peer), + RbcServiceMessage::AdvanceLocalRound { round, reply } => { + let result = self + .kernel + .advance_local_round(round) + .map_err(RbcServiceError::from); + let _ = reply.send(result); + } + RbcServiceMessage::RetryHeaders(reply) => { + self.retry_pending_headers(); + let _ = reply.send(()); + } + } + } + + fn start_local_header( + &mut self, + header: RbcLocalHeader, + ) -> Result { + self.kernel.advance_local_round(header.round)?; + let local = self.kernel.start_local_initial_header( + header.round, + header.block_references, + header.acknowledgment_references, + header.meta_creation_time_ns, + header.transactions_commitment, + )?; + let canonical = local.header().clone(); + let proposals = self.make_initial_proposals(&local); + let (pinned, effects) = local.into_parts(); + + self.notify_header_staged(pinned); + for (recipient, proposal) in proposals { + self.retained_initials + .insert((canonical.reference(), recipient), proposal.clone()); + self.send_network(recipient, NetworkMessage::RbcInitial(proposal)); + } + self.process_effects(effects); + Ok(canonical) + } + + fn make_initial_proposals( + &self, + local: &RbcLocalInitial, + ) -> Vec<(AuthorityIndex, RbcHeaderProposal)> { + let header = local.header().clone(); + match &self.initial_authenticator { + RbcInitialAuthenticator::Ed25519(signer) => { + let digest = self + .kernel + .make_local_initial_signature_digest(local) + .expect("local RBC handle must remain selected"); + let proof = RbcInitialProof::Ed25519(signer.sign_digest(&digest)); + self.public_initial_proposals(header, proof) + } + RbcInitialAuthenticator::MlDsa44(signer) => { + let digest = self + .kernel + .make_local_initial_signature_digest(local) + .expect("local RBC handle must remain selected"); + let proof = + RbcInitialProof::MlDsa44(signer.sign_digest(&BlockDigest::from(digest))); + self.public_initial_proposals(header, proof) + } + RbcInitialAuthenticator::MlDsa65(signer) => { + let digest = self + .kernel + .make_local_initial_signature_digest(local) + .expect("local RBC handle must remain selected"); + let proof = + RbcInitialProof::MlDsa65(signer.sign_digest(&BlockDigest::from(digest))); + self.public_initial_proposals(header, proof) + } + RbcInitialAuthenticator::Mac => self + .committee + .authorities() + .filter(|recipient| *recipient != self.own_authority) + .map(|recipient| { + let tag = self + .kernel + .make_local_initial_mac_tag(local, recipient) + .expect("local RBC handle must remain selected"); + ( + recipient, + RbcHeaderProposal::new(header.clone(), RbcInitialProof::Mac(tag)), + ) + }) + .collect(), + } + } + + fn public_initial_proposals( + &self, + header: RbcCanonicalHeader, + proof: RbcInitialProof, + ) -> Vec<(AuthorityIndex, RbcHeaderProposal)> { + self.committee + .authorities() + .filter(|recipient| *recipient != self.own_authority) + .map(|recipient| { + ( + recipient, + RbcHeaderProposal::new(header.clone(), proof.clone()), + ) + }) + .collect() + } + + fn accept_direct_initial(&mut self, peer: AuthorityIndex, proposal: RbcHeaderProposal) { + let (header, proof) = proposal.into_parts(); + let block_ref = header.reference(); + match self + .kernel + .accept_direct_initial_header(peer, header, &proof) + { + Ok(RbcInitialHeaderOutcome::Authenticated { effects }) => { + self.finish_header_staging(block_ref, Some(peer)); + self.process_effects(effects); + } + Ok(RbcInitialHeaderOutcome::StagedUnauthenticated { effects, error }) => { + self.finish_header_staging(block_ref, Some(peer)); + self.process_effects(effects); + self.reject(Some(peer), error.into()); + } + Err(error) => self.reject(Some(peer), error.into()), + } + } + + fn answer_header_request(&mut self, peer: AuthorityIndex, block_ref: BlockReference) { + if !self.committee.known_authority(peer) { + self.reject(Some(peer), RbcError::UnknownAuthority(peer).into()); + return; + } + if peer == self.own_authority { + self.reject(Some(peer), RbcError::LoopbackPhase.into()); + return; + } + match self.kernel.pinned_header(block_ref) { + Ok(Some(header)) => self.send_network( + peer, + NetworkMessage::RbcHeaderResponse(header.header().clone()), + ), + Ok(None) => {} + Err(error) => self.reject(Some(peer), error.into()), + } + } + + fn accept_header_response(&mut self, peer: AuthorityIndex, header: RbcCanonicalHeader) { + let block_ref = header.reference(); + if !self.committee.known_authority(peer) { + self.reject(Some(peer), RbcError::UnknownAuthority(peer).into()); + return; + } + if peer == self.own_authority { + self.reject(Some(peer), RbcError::LoopbackPhase.into()); + return; + } + let Some(fetch) = self.pending_fetches.get(&block_ref) else { + self.reject( + Some(peer), + RbcServiceError::UnexpectedHeaderResponse(block_ref), + ); + return; + }; + if !fetch.holders.contains(peer) { + self.reject( + Some(peer), + RbcServiceError::HeaderResponseFromNonHolder { block_ref, peer }, + ); + return; + } + + match self.kernel.accept_recovered_header(header) { + Ok(effects) => { + self.finish_header_staging(block_ref, Some(peer)); + self.process_effects(effects); + } + Err(error) => self.reject(Some(peer), error.into()), + } + } + + fn finish_header_staging(&mut self, block_ref: BlockReference, peer: Option) { + match self.kernel.pinned_header(block_ref) { + Ok(Some(header)) => { + self.pending_fetches.remove(&block_ref); + self.notify_header_staged(header); + } + Ok(None) => self.reject(peer, RbcError::HeaderUnavailable(block_ref).into()), + Err(error) => self.reject(peer, error.into()), + } + } + + fn notify_header_staged(&mut self, header: PinnedRbcHeader) { + if self.staged_notifications.insert(header.reference()) { + let _ = self.events.send(RbcServiceEvent::HeaderStaged(header)); + } + } + + fn process_effects(&mut self, effects: Vec) { + for effect in effects { + match effect { + RbcEffect::MulticastPhase { phase, block_ref } => { + self.retained_phases.insert((block_ref, phase)); + let recipients: Vec<_> = self + .committee + .authorities() + .filter(|recipient| *recipient != self.own_authority) + .collect(); + for recipient in recipients { + match self.kernel.make_phase_message(phase, block_ref, recipient) { + Ok(message) => { + self.send_network(recipient, NetworkMessage::RbcPhase(message)) + } + Err(error) => self.reject(None, error.into()), + } + } + } + RbcEffect::NeedHeader { block_ref, holders } => { + self.note_pending_fetch(block_ref, holders); + } + RbcEffect::Deliver(header) => { + self.pending_fetches.remove(&header.reference()); + let _ = self.events.send(RbcServiceEvent::Delivered(header)); + } + } + } + } + + fn note_pending_fetch(&mut self, block_ref: BlockReference, holders: AuthoritySet) { + self.pending_fetches + .entry(block_ref) + .and_modify(|fetch| fetch.holders |= holders) + .or_insert(PendingHeaderFetch { + holders, + requested_from: AuthoritySet::default(), + }); + self.send_fetch_wave(block_ref); + } + + fn send_fetch_wave(&mut self, block_ref: BlockReference) { + let Some(fetch) = self.pending_fetches.get_mut(&block_ref) else { + return; + }; + let eligible: Vec<_> = fetch + .holders + .present() + .filter(|peer| *peer != self.own_authority && self.connected_peers.contains(*peer)) + .collect(); + let mut recipients: Vec<_> = eligible + .iter() + .copied() + .filter(|peer| !fetch.requested_from.contains(*peer)) + .take(HEADER_REQUEST_FANOUT) + .collect(); + if recipients.is_empty() && !eligible.is_empty() { + fetch.requested_from.clear(); + recipients.extend(eligible.into_iter().take(HEADER_REQUEST_FANOUT)); + } + for recipient in &recipients { + fetch.requested_from.insert(*recipient); + } + for recipient in recipients { + self.send_network(recipient, NetworkMessage::RbcHeaderRequest(block_ref)); + } + } + + fn retry_pending_headers(&mut self) { + let block_refs: Vec<_> = self.pending_fetches.keys().copied().collect(); + for block_ref in block_refs { + match self.kernel.retry_header_request(block_ref) { + Ok(Some(RbcEffect::NeedHeader { holders, .. })) => { + if let Some(fetch) = self.pending_fetches.get_mut(&block_ref) { + fetch.holders |= holders; + } + self.send_fetch_wave(block_ref); + } + Ok(Some(_)) => unreachable!("header retry can only request a header"), + Ok(None) => { + self.pending_fetches.remove(&block_ref); + } + Err(error) => { + self.pending_fetches.remove(&block_ref); + self.reject(None, error.into()); + } + } + } + } + + fn peer_connected(&mut self, peer: AuthorityIndex) { + if !self.committee.known_authority(peer) { + self.reject(Some(peer), RbcError::UnknownAuthority(peer).into()); + return; + } + if peer == self.own_authority { + self.reject(Some(peer), RbcError::LoopbackPhase.into()); + return; + } + self.connected_peers.insert(peer); + + let initials: Vec<_> = self + .retained_initials + .iter() + .filter_map(|((_, recipient), proposal)| { + (*recipient == peer).then_some(proposal.clone()) + }) + .collect(); + for proposal in initials { + self.send_network(peer, NetworkMessage::RbcInitial(proposal)); + } + + let phases: Vec<_> = self.retained_phases.iter().copied().collect(); + for (block_ref, phase) in phases { + match self.kernel.make_phase_message(phase, block_ref, peer) { + Ok(message) => self.send_network(peer, NetworkMessage::RbcPhase(message)), + Err(error) => self.reject(Some(peer), error.into()), + } + } + + let pending: Vec<_> = self + .pending_fetches + .iter() + .filter_map(|(block_ref, fetch)| fetch.holders.contains(peer).then_some(*block_ref)) + .collect(); + for block_ref in pending { + if let Some(fetch) = self.pending_fetches.get_mut(&block_ref) { + fetch.requested_from.insert(peer); + } + self.send_network(peer, NetworkMessage::RbcHeaderRequest(block_ref)); + } + } + + fn peer_disconnected(&mut self, peer: AuthorityIndex) { + if !self.committee.known_authority(peer) { + self.reject(Some(peer), RbcError::UnknownAuthority(peer).into()); + return; + } + if peer == self.own_authority { + self.reject(Some(peer), RbcError::LoopbackPhase.into()); + return; + } + self.connected_peers.remove(peer); + } + + fn send_network(&self, recipient: AuthorityIndex, message: NetworkMessage) { + let _ = self + .events + .send(RbcServiceEvent::Network { recipient, message }); + } + + fn reject(&self, peer: Option, error: RbcServiceError) { + let _ = self.events.send(RbcServiceEvent::Rejected { peer, error }); + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::{ + crypto::{ + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + }, + starfish_rbc::RbcPhase, + types::VerifiedBlock, + }; + + fn instance() -> RbcProtocolInstanceId { + RbcProtocolInstanceId::new([0x51; 32]).unwrap() + } + + fn local_header(round: RoundNumber, committee_size: AuthorityIndex) -> RbcLocalHeader { + RbcLocalHeader { + round, + block_references: (0..committee_size) + .map(|authority| BlockReference::new_test(authority, round - 1)) + .collect(), + acknowledgment_references: Vec::new(), + meta_creation_time_ns: 17, + transactions_commitment: TransactionsCommitment::default(), + } + } + + fn start_service( + own_authority: AuthorityIndex, + scheme: BlockAuthenticationScheme, + ) -> ( + RbcServiceHandle, + mpsc::UnboundedReceiver, + JoinHandle<()>, + ) { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let authenticator = match scheme { + BlockAuthenticationScheme::Ed25519 => RbcInitialAuthenticator::Ed25519(dummy_signer()), + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, + BlockAuthenticationScheme::MlDsa44 => { + RbcInitialAuthenticator::MlDsa44(dummy_ml_dsa_44_signer()) + } + BlockAuthenticationScheme::MlDsa65 => { + RbcInitialAuthenticator::MlDsa65(dummy_ml_dsa_65_signer()) + } + }; + start_starfish_rbc_service( + committee, + own_authority, + instance(), + scheme, + Arc::new(keyrings[own_authority as usize].clone()), + authenticator, + 1, + Duration::from_secs(3_600), + ) + .unwrap() + } + + async fn next_event(events: &mut mpsc::UnboundedReceiver) -> RbcServiceEvent { + tokio::time::timeout(Duration::from_secs(2), events.recv()) + .await + .expect("service event timed out") + .expect("service stopped unexpectedly") + } + + #[tokio::test] + async fn local_initial_authenticator_wiring_covers_all_four_modes() { + for scheme in [ + BlockAuthenticationScheme::Ed25519, + BlockAuthenticationScheme::MlDsa44, + BlockAuthenticationScheme::MlDsa65, + BlockAuthenticationScheme::MacVector, + ] { + let (handle, mut events, task) = start_service(0, scheme); + let canonical = handle.start_local_header(local_header(1, 4)).await.unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::HeaderStaged(ref header) + if header.reference() == canonical.reference() + )); + let RbcServiceEvent::Network { + recipient: 1, + message: NetworkMessage::RbcInitial(proposal), + } = next_event(&mut events).await + else { + panic!("expected first recipient's INIT") + }; + let proof_matches = matches!( + (scheme, proposal.proof()), + ( + BlockAuthenticationScheme::Ed25519, + RbcInitialProof::Ed25519(_) + ) | ( + BlockAuthenticationScheme::MlDsa44, + RbcInitialProof::MlDsa44(_) + ) | ( + BlockAuthenticationScheme::MlDsa65, + RbcInitialProof::MlDsa65(_) + ) | ( + BlockAuthenticationScheme::MacVector, + RbcInitialProof::Mac(_) + ) + ); + assert!(proof_matches, "wrong INIT proof for {scheme:?}"); + + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut receiver = StarfishRbcKernel::new( + committee, + 1, + instance(), + scheme, + Arc::new(keyrings[1].clone()), + 1, + ) + .unwrap(); + assert!(matches!( + receiver + .accept_direct_initial_header(0, proposal.header().clone(), proposal.proof(),) + .unwrap(), + RbcInitialHeaderOutcome::Authenticated { .. } + )); + + // Two remaining INITs and three recipient-specific ECHOs. + for _ in 0..5 { + let _ = next_event(&mut events).await; + } + drop(handle); + task.await.unwrap(); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_local_start_waits_for_kernel_selection_and_event_enqueue() { + let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); + let blocking_handle = handle.clone(); + let canonical = tokio::task::spawn_blocking(move || { + blocking_handle.start_local_header_blocking(local_header(1, 4)) + }) + .await + .unwrap() + .unwrap(); + + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::HeaderStaged(ref header) + if header.reference() == canonical.reference() + )); + drop(handle); + task.await.unwrap(); + } + + #[tokio::test] + async fn local_start_advances_the_admission_window_before_pinning() { + let (handle, events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); + + // The kernel starts at round 1, whose ordinary future-admission window + // ends at round 101. A locally selected proposal is trusted progress + // and must move that window before its own reference is validated. + let canonical = handle + .start_local_header(local_header(150, 4)) + .await + .unwrap(); + assert_eq!(canonical.reference().round, 150); + + drop(handle); + drop(events); + task.await.unwrap(); + } + + #[tokio::test] + async fn local_mac_start_materializes_distinct_recipient_messages() { + let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::MacVector); + let canonical = handle.start_local_header(local_header(1, 4)).await.unwrap(); + + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::HeaderStaged(ref header) if header.reference() == canonical.reference() + )); + + let mut initial_proofs = Vec::new(); + for expected_recipient in 1..4 { + let RbcServiceEvent::Network { recipient, message } = next_event(&mut events).await + else { + panic!("expected initial network event") + }; + assert_eq!(recipient, expected_recipient); + let NetworkMessage::RbcInitial(proposal) = message else { + panic!("expected initial proposal") + }; + assert_eq!(proposal.header(), &canonical); + let RbcInitialProof::Mac(tag) = proposal.proof() else { + panic!("MAC mode must send one tag") + }; + initial_proofs.push(*tag); + } + assert!(initial_proofs.windows(2).all(|pair| pair[0] != pair[1])); + + let mut phases = Vec::new(); + for expected_recipient in 1..4 { + let RbcServiceEvent::Network { recipient, message } = next_event(&mut events).await + else { + panic!("expected phase network event") + }; + assert_eq!(recipient, expected_recipient); + let NetworkMessage::RbcPhase(message) = message else { + panic!("expected RBC phase") + }; + assert_eq!(message.phase(), RbcPhase::Echo); + assert_eq!(message.recipient(), recipient); + assert_eq!(message.sender(), 0); + phases.push(message); + } + assert!(phases.windows(2).all(|pair| pair[0] != pair[1])); + + // The initial proposal and authorized phase intent survive a missing + // connection. Reconnection replays INIT first and rematerializes a + // fresh recipient-specific phase message from the kernel intent. + handle.peer_connected(2).unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient: 2, + message: NetworkMessage::RbcInitial(ref proposal), + } if proposal.header() == &canonical + )); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient: 2, + message: NetworkMessage::RbcPhase(ref message), + } if message.phase() == RbcPhase::Echo && message.recipient() == 2 + )); + + handle.header_request(1, canonical.reference()).unwrap(); + let RbcServiceEvent::Network { recipient, message } = next_event(&mut events).await else { + panic!("expected header response") + }; + assert_eq!(recipient, 1); + assert!(matches!( + message, + NetworkMessage::RbcHeaderResponse(header) if header == canonical + )); + + drop(handle); + task.await.unwrap(); + } + + fn echo_messages_for( + canonical: &RbcCanonicalHeader, + recipient: AuthorityIndex, + ) -> Vec<(AuthorityIndex, RbcPhaseMessage)> { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let mut author = StarfishRbcKernel::new( + committee.clone(), + 0, + instance(), + BlockAuthenticationScheme::Ed25519, + Arc::new(keyrings[0].clone()), + 1, + ) + .unwrap(); + let local = author + .start_local_initial_header( + canonical.reference().round, + canonical.block_references().to_vec(), + canonical.acknowledgment_references(), + canonical.meta_creation_time_ns(), + canonical.transactions_commitment(), + ) + .unwrap(); + let digest = author.make_local_initial_signature_digest(&local).unwrap(); + let proof = RbcInitialProof::Ed25519(dummy_signer().sign_digest(&digest)); + drop(local.into_parts()); + + let mut messages = vec![( + 0, + author + .make_phase_message(RbcPhase::Echo, canonical.reference(), recipient) + .unwrap(), + )]; + for sender in 1..=2 { + let mut kernel = StarfishRbcKernel::new( + committee.clone(), + sender, + instance(), + BlockAuthenticationScheme::Ed25519, + Arc::new(keyrings[sender as usize].clone()), + 1, + ) + .unwrap(); + let outcome = kernel + .accept_direct_initial_header(0, canonical.clone(), &proof) + .unwrap(); + assert!(matches!( + outcome, + RbcInitialHeaderOutcome::Authenticated { .. } + )); + messages.push(( + sender, + kernel + .make_phase_message(RbcPhase::Echo, canonical.reference(), recipient) + .unwrap(), + )); + } + messages + } + + #[tokio::test] + async fn header_recovery_is_durable_and_fans_out_to_new_holders() { + let (handle, mut events, task) = start_service(3, BlockAuthenticationScheme::Ed25519); + for peer in 0..3 { + handle.peer_connected(peer).unwrap(); + } + + let canonical = RbcCanonicalHeader::try_new( + 0, + 1, + (0..4) + .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) + .collect(), + Vec::new(), + 23, + TransactionsCommitment::default(), + ) + .unwrap(); + for (peer, message) in echo_messages_for(&canonical, 3) { + handle.phase(peer, message).unwrap(); + } + + let mut first_wave = Vec::new(); + for _ in 0..2 { + let RbcServiceEvent::Network { recipient, message } = next_event(&mut events).await + else { + panic!("expected header request") + }; + assert!(matches!( + message, + NetworkMessage::RbcHeaderRequest(block_ref) if block_ref == canonical.reference() + )); + first_wave.push(recipient); + } + assert_eq!(first_wave, vec![0, 1]); + + let unrelated = RbcCanonicalHeader::try_new( + 1, + 1, + canonical.block_references().to_vec(), + Vec::new(), + 24, + TransactionsCommitment::default(), + ) + .unwrap(); + handle.header_response(0, unrelated).unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Rejected { + peer: Some(0), + error: RbcServiceError::UnexpectedHeaderResponse(_), + } + )); + + // Replacing a connection does not own or discard the fetch. The + // central actor immediately retries the still-pending reference when + // the untried authenticated holder reconnects. + handle.peer_disconnected(2).unwrap(); + handle.peer_connected(2).unwrap(); + let RbcServiceEvent::Network { recipient, message } = next_event(&mut events).await else { + panic!("expected reconnect retry to the untried holder") + }; + assert_eq!(recipient, 2); + assert!(matches!( + message, + NetworkMessage::RbcHeaderRequest(block_ref) if block_ref == canonical.reference() + )); + + handle.header_response(2, canonical.clone()).unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::HeaderStaged(ref header) if header.reference() == canonical.reference() + )); + for expected_recipient in 0..3 { + let RbcServiceEvent::Network { recipient, message } = next_event(&mut events).await + else { + panic!("expected READY after recovery") + }; + assert_eq!(recipient, expected_recipient); + assert!(matches!( + message, + NetworkMessage::RbcPhase(ref phase) + if phase.phase() == RbcPhase::Ready + && phase.recipient() == expected_recipient + )); + } + + handle.retry_headers().await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(20), events.recv()) + .await + .is_err() + ); + + drop(handle); + task.await.unwrap(); + } + + #[test] + fn service_rejects_mismatched_initial_authenticator() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let result = start_starfish_rbc_service( + committee, + 0, + instance(), + BlockAuthenticationScheme::Ed25519, + Arc::new(keyrings[0].clone()), + RbcInitialAuthenticator::Mac, + 1, + Duration::from_secs(1), + ); + assert!(matches!( + result, + Err(RbcServiceError::InitialAuthenticatorSchemeMismatch { .. }) + )); + } +} diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 9ef887cb..6892fc9c 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -19,6 +19,8 @@ use crate::{ metrics::Metrics, runtime::timestamp_utc, sailfish_service::SailfishServiceMessage, + starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, + starfish_rbc_service::{RbcLocalHeader, RbcServiceHandle}, types::{ AuthorityIndex, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishNoVoteCert, SailfishTimeoutCert, Stake, @@ -64,6 +66,7 @@ pub struct Syncer { pub(crate) metrics: Arc, bls_tx: Option>, sailfish_tx: Option>, + starfish_rbc_service: Option, } pub trait SyncerSignals: Send + Sync { @@ -95,6 +98,7 @@ impl Syncer { metrics: Arc, bls_tx: Option>, sailfish_tx: Option>, + starfish_rbc_service: Option, ) -> Self { let committee_size = core.committee().len(); let own_stake = core @@ -114,6 +118,7 @@ impl Syncer { metrics, bls_tx, sailfish_tx, + starfish_rbc_service, } } @@ -147,6 +152,11 @@ impl Syncer { if success { tracing::debug!("Attempt to create block from syncer after adding block"); self.try_new_block(BlockCreationReason::NewBlocks); + if self.core.dag_state().consensus_protocol.is_starfish_rbc() { + // A previously delivered header may have become dirty-DAG + // connected and clean during insertion. + self.try_new_commit(); + } } ( pending_blocks_with_transactions, @@ -175,6 +185,9 @@ impl Syncer { if success { tracing::debug!("Attempt to create block from syncer after adding headers"); self.try_new_block(BlockCreationReason::NewHeaders); + if self.core.dag_state().consensus_protocol.is_starfish_rbc() { + self.try_new_commit(); + } } (missing_parents, processed_refs) } @@ -204,6 +217,24 @@ impl Syncer { } } + /// Called after the local Starfish-RBC service delivers exact header + /// references. Delivery is separate from dirty insertion and transaction + /// availability; any newly dependency-closed vertices can immediately + /// unblock both proposal and commit paths. + pub fn apply_starfish_rbc_deliveries(&mut self, delivered_headers: Vec) { + let previous_rounds = self.capture_rounds(); + if self + .core + .dag_state() + .apply_starfish_rbc_deliveries(&delivered_headers) + { + self.maybe_update_proposal_wait(); + self.maybe_signal_proposal_round_advance(previous_rounds); + self.try_new_block(BlockCreationReason::CertificateEvent); + self.try_new_commit(); + } + } + /// Store a Sailfish++ timeout certificate in DagState and retry block /// creation (a TC may unblock block creation for the next round). pub fn apply_timeout_cert(&mut self, cert: SailfishTimeoutCert) { @@ -306,6 +337,21 @@ impl Syncer { tracing::debug!("Attempt to create new block in syncer after one trigger"); let previous_rounds = self.capture_rounds(); if let Some(ref block) = self.core.try_new_block(reason.as_str()) { + if self.core.dag_state().consensus_protocol.is_starfish_rbc() { + let canonical = RbcCanonicalHeader::from_block_header(block.header()) + .expect("locally built Starfish-RBC block must have canonical header content"); + let selected = self + .starfish_rbc_service + .as_ref() + .expect("Starfish-RBC protocol must start its RBC service") + .start_local_header_blocking(RbcLocalHeader::from_canonical(&canonical)) + .expect("local Starfish-RBC header must be accepted before dissemination"); + assert_eq!( + selected.reference(), + *block.reference(), + "RBC service selected a different local header reference" + ); + } if let Some(started_at) = self.proposal_wait_started_at.take() { self.metrics .proposal_wait_time_total_us @@ -584,7 +630,7 @@ mod tests { assert_eq!(core.dag_state().proposal_round(), 3); assert_eq!(core.last_proposed(), 0); - let mut syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None); + let mut syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None, None); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); syncer.recompute_subscriber_stake(); @@ -681,6 +727,7 @@ mod tests { metrics, None, None, + None, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index c988c668..264cb171 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -48,6 +48,7 @@ use crate::{ dag_state::ConsensusProtocol, data::{Data, IN_MEMORY_BLOCKS, IN_MEMORY_BLOCKS_BYTES}, encoder::ShardEncoder, + starfish_rbc::RbcCanonicalHeader, threshold_clock::threshold_clock_valid_block_header, }; @@ -833,6 +834,66 @@ pub struct VerifiedBlock { } impl VerifiedBlock { + /// Construct the authentication-free block carrier used by Starfish-RBC. + /// + /// The header identity is the canonical Starfish-RBC content digest. The + /// author proof is carried separately by the RBC INIT message and is never + /// serialized into this block header. Transaction data remains unchanged + /// from plain Starfish so the existing shard layer can be reused. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new_starfish_rbc( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions: Vec, + encoded_transactions: Option>, + ) -> Self { + let transactions_commitment = if let Some(ref encoded) = encoded_transactions { + TransactionsCommitment::new_from_encoded_transactions(encoded, authority as usize).0 + } else { + TransactionsCommitment::default() + }; + let (intersection, extra_references) = + compress_acknowledgments(&block_references, &acknowledgment_references); + let logical_acknowledgments = + expand_acknowledgments(&block_references, intersection, &extra_references); + let digest = BlockDigest::new_starfish_rbc_header( + authority, + round, + &block_references, + &logical_acknowledgments, + meta_creation_time_ns, + transactions_commitment, + ); + let transaction_data = + (!transactions.is_empty()).then(|| TransactionData::new(transactions)); + Self { + header: BlockHeader { + reference: BlockReference { + authority, + round, + digest, + }, + block_references, + meta_creation_time_ns, + authentication: BlockAuthentication::None, + transactions_commitment: Some(transactions_commitment), + ack: Some(AckFields { + intersection, + extra_references, + }), + strong_vote: None, + bls: None, + sailfish: None, + unprovable_certificate: None, + serialized: None, + }, + transaction_data, + } + } + pub fn new( authority: AuthorityIndex, round: RoundNumber, @@ -1440,6 +1501,20 @@ impl VerifiedBlock { authentication_scheme: BlockAuthenticationScheme, mac_keys: &[MacKey], ) -> eyre::Result> { + if consensus_protocol.is_starfish_rbc() { + ensure!( + matches!(&self.header.authentication, BlockAuthentication::None), + "Starfish-RBC block carriers must not embed authentication" + ); + let (shard, _) = + self.verify_transactions(committee, own_id, encoder, consensus_protocol)?; + let canonical = RbcCanonicalHeader::from_block_header(&self.header) + .map_err(|error| eyre::eyre!(error))?; + canonical + .validate_for_committee(committee) + .map_err(|error| eyre::eyre!(error))?; + return Ok(shard); + } let (shard, digest_transactions_commitment) = self.verify_transactions(committee, own_id, encoder, consensus_protocol)?; self.verify_block_structure( @@ -1830,7 +1905,9 @@ impl VerifiedBlock { ); } } - ConsensusProtocol::Starfish | ConsensusProtocol::StarfishSpeed => { + ConsensusProtocol::Starfish + | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishSpeed => { ensure!( threshold_clock_valid_block_header(&self.header, committee), "Threshold clock is not valid" @@ -3061,6 +3138,84 @@ mod tests { assert_eq!(block.acknowledgments(), vec![c, d]); } + #[test] + fn starfish_rbc_carrier_uses_canonical_content_identity_without_authentication() { + let parents = vec![ + BlockReference::new_test(0, 1), + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ]; + let extra = BlockReference::new_test(3, 1); + let raw_acknowledgments = vec![extra, parents[2]]; + let block = VerifiedBlock::new_starfish_rbc( + 0, + 2, + parents.clone(), + raw_acknowledgments, + 17, + Vec::new(), + None, + ); + + assert_eq!(block.authentication(), &BlockAuthentication::None); + assert_eq!(block.acknowledgments(), vec![parents[2], extra]); + let commitment = block + .header() + .transactions_commitment + .expect("RBC carrier must commit its Starfish payload"); + assert_eq!( + block.digest(), + BlockDigest::new_starfish_rbc_header( + 0, + 2, + &parents, + &[parents[2], extra], + 17, + commitment, + ) + ); + } + + #[test] + fn starfish_rbc_carrier_verification_is_content_only() { + let committee = Committee::new_for_benchmarks(4); + let parents: Vec<_> = committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(); + let mut block = + VerifiedBlock::new_starfish_rbc(0, 1, parents, Vec::new(), 19, Vec::new(), None); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + assert!( + block + .verify_with_authentication( + &committee, + 0, + 0, + &mut encoder, + ConsensusProtocol::StarfishRbc, + BlockAuthenticationScheme::Ed25519, + &[], + ) + .is_ok() + ); + + block.header.authentication = BlockAuthentication::Ed25519(SignatureBytes::default()); + assert!( + block + .verify_with_authentication( + &committee, + 0, + 0, + &mut encoder, + ConsensusProtocol::StarfishRbc, + BlockAuthenticationScheme::Ed25519, + &[], + ) + .is_err() + ); + } + #[test] fn falls_back_to_legacy_ack_encoding_when_suffix_index_exceeds_u8() { let block_references: Vec<_> = (0..300) diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 123c872d..65ac2a8f 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -24,6 +24,7 @@ use crate::{ network::Network, prometheus, runtime::{JoinError, JoinHandle}, + starfish_rbc::RbcProtocolInstanceId, transactions_generator::TransactionGenerator, types::{AuthorityIndex, BlockAuthenticationScheme, PartialSig}, }; @@ -50,6 +51,24 @@ impl Validator { public_config.parameters.block_authentication.as_deref(), ) .map_err(|error| eyre!(error))?; + let is_starfish_rbc = protocol_config.consensus_protocol.is_starfish_rbc(); + if is_starfish_rbc { + let protocol_instance = public_config + .parameters + .starfish_rbc_protocol_instance + .ok_or_else(|| eyre!("Starfish-RBC protocol instance is missing"))?; + RbcProtocolInstanceId::new(protocol_instance).map_err(|error| eyre!(error))?; + } + if (is_starfish_rbc + || protocol_config.block_authentication_scheme == BlockAuthenticationScheme::MacVector) + && private_config.mac_keys.len() != committee.len() + { + return Err(eyre!( + "MAC keyring length {} does not match committee size {}", + private_config.mac_keys.len(), + committee.len(), + )); + } match protocol_config.block_authentication_scheme { BlockAuthenticationScheme::Ed25519 => { if committee.get_public_key(authority) != Some(&private_config.keypair.public_key()) @@ -59,15 +78,7 @@ impl Validator { )); } } - BlockAuthenticationScheme::MacVector => { - if private_config.mac_keys.len() != committee.len() { - return Err(eyre!( - "MAC keyring length {} does not match committee size {}", - private_config.mac_keys.len(), - committee.len(), - )); - } - } + BlockAuthenticationScheme::MacVector => {} BlockAuthenticationScheme::MlDsa44 => { if committee.get_ml_dsa_44_public_key(authority) != Some(&private_config.ml_dsa_44_keypair.public_key()) @@ -218,7 +229,8 @@ impl Validator { partial_sig_rx, bls_cert_aggregator, bls_signer_for_service, - ); + ) + .await; tracing::info!("Validator {authority} listening on {network_address}"); tracing::info!("Validator {authority} exposing metrics on {metrics_address}"); @@ -313,6 +325,11 @@ mod smoke_tests { let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); + if consensus == "starfish-rbc" { + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + } let parameters = Parameters::default(); let dir = TempDir::new().unwrap(); @@ -341,7 +358,11 @@ mod smoke_tests { .all_metric_addresses() .map(|a| a.to_owned()) .collect(); - let timeout = config::param_defaults::default_leader_timeout() * 5; + // Four RBC authentication variants run in parallel in the full test + // suite and include expensive ML-DSA signing. Give that composed flow + // enough scheduling headroom without relaxing existing protocols. + let timeout_multiplier = if consensus == "starfish-rbc" { 20 } else { 5 }; + let timeout = config::param_defaults::default_leader_timeout() * timeout_multiplier; tokio::select! { _ = await_for_commits(addresses) => (), @@ -382,6 +403,10 @@ mod smoke_tests { #[test_case("bluestreak-mac", None, 920)] #[test_case("bluestreak", Some("ml-dsa-44"), 940)] #[test_case("bluestreak", Some("ml-dsa-65"), 1120)] + #[test_case("starfish-rbc", None, 1400)] + #[test_case("starfish-rbc", Some("mac"), 1440)] + #[test_case("starfish-rbc", Some("ml-dsa-44"), 1480)] + #[test_case("starfish-rbc", Some("ml-dsa-65"), 1520)] #[tokio::test] async fn validator_commit( consensus: &str, @@ -396,12 +421,53 @@ mod smoke_tests { run_commit_test("bluestreak", None, 150).await; } + #[tokio::test] + async fn starfish_rbc_single_validator_starts_on_current_thread_runtime() { + let committee_size = 4; + // Give the sole running validator quorum stake so startup immediately + // exercises local RBC proposal construction before any peer connects. + let committee = Committee::new_test(vec![100, 1, 1, 1]); + let mut public_config = + NodePublicConfig::new_for_tests(committee_size).with_port_offset(1600); + public_config.parameters.block_authentication = Some("mac".to_string()); + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + + let dir = TempDir::new().unwrap(); + let private_config = + NodePrivateConfig::new_for_benchmarks(dir.as_ref(), committee_size).remove(0); + fs::create_dir_all(&private_config.storage_path).unwrap(); + + let validator = time::timeout( + Duration::from_secs(5), + Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ), + ) + .await + .expect("Starfish-RBC startup must not block its async runtime") + .unwrap(); + validator.stop().await; + } + async fn run_sync_test(consensus: &str, block_authentication: Option<&str>, port_offset: u16) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); + if consensus == "starfish-rbc" { + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + } let parameters = Parameters::default(); let dir = TempDir::new().unwrap(); @@ -499,6 +565,7 @@ mod smoke_tests { #[test_case("bluestreak-mac", None, 960)] #[test_case("bluestreak", Some("ml-dsa-44"), 980)] #[test_case("bluestreak", Some("ml-dsa-65"), 1260)] + #[test_case("starfish-rbc", Some("mac"), 1560)] #[tokio::test] async fn validator_sync(consensus: &str, block_authentication: Option<&str>, port_offset: u16) { run_sync_test(consensus, block_authentication, port_offset).await; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 73a6288c..ed651520 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -70,7 +70,7 @@ enum Operation { consensus: String, /// Block signature scheme. Defaults to Ed25519 and is not applicable /// to the experimental `*-mac` protocols. - #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65")] + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, }, /// Deploy a local validator for test. Dryrun mode uses @@ -103,7 +103,7 @@ enum Operation { consensus: String, /// Block signature scheme. Defaults to Ed25519 and is not applicable /// to the experimental `*-mac` protocols. - #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65")] + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] @@ -158,7 +158,7 @@ enum Operation { consensus: String, /// Block signature scheme. Defaults to Ed25519 and is not applicable /// to the experimental `*-mac` protocols. - #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65")] + #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, @@ -268,6 +268,9 @@ async fn main() -> Result<()> { node_parameters.adversarial_latency = adversarial_latency; node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.block_authentication = block_authentication; + if consensus_protocol == "starfish-rbc" { + node_parameters.refresh_starfish_rbc_protocol_instance(); + } if let Some(ref mode) = dissemination_mode { node_parameters.dissemination_mode = parse_dissemination_mode(mode)?; } @@ -308,13 +311,16 @@ fn benchmark_genesis( tracing::info!("Generated committee file: {}", committee_path.display()); // Generate the public node config file. - let node_parameters = match node_parameters_path { + let mut node_parameters = match node_parameters_path { Some(path) => NodeParameters::load(&path).wrap_err(format!( "Failed to load parameters file '{}'", path.display() ))?, None => NodeParameters::default(), }; + if node_parameters.starfish_rbc_protocol_instance.is_none() { + node_parameters.refresh_starfish_rbc_protocol_instance(); + } let node_public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters)); let mut node_public_config_path = working_directory.clone(); @@ -788,4 +794,30 @@ mod tests { assert_eq!(consensus, "mysticeti"); assert_eq!(block_authentication.as_deref(), Some("ml-dsa-65")); } + + #[test] + fn local_benchmark_parses_starfish_rbc_mac_authentication() { + let args = Args::try_parse_from([ + "starfish", + "local-benchmark", + "--committee-size", + "4", + "--consensus", + "starfish-rbc", + "--block-authentication", + "mac", + ]) + .unwrap(); + + let Operation::LocalBenchmark { + consensus, + block_authentication, + .. + } = args.operation + else { + panic!("expected local-benchmark operation"); + }; + assert_eq!(consensus, "starfish-rbc"); + assert_eq!(block_authentication.as_deref(), Some("mac")); + } } diff --git a/docs/starfish-rbc-protocol.md b/docs/starfish-rbc-protocol.md index e7eb4d8f..72a7ac03 100644 --- a/docs/starfish-rbc-protocol.md +++ b/docs/starfish-rbc-protocol.md @@ -1,7 +1,7 @@ # Starfish-RBC protocol specification -Status: protocol specification with isolated RBC kernel and canonical header staging; network and -DAG integration pending +Status: integrated benchmark prototype; extended adversarial validation and performance comparison +pending This document specifies the first correctness-oriented prototype of Starfish with reliable header certification and a signature-free MAC configuration. The provisional CLI name is `starfish-rbc`. @@ -468,9 +468,11 @@ Every honest ECHO and READY sender is a header holder. Retrieval therefore proce `NeedHeader` is a recovery wake-up, not a one-shot delivery assumption. The kernel re-emits it when the authenticated holder set grows and exposes the current effect to a durable retry timer. The -integration layer must keep that timer active, retry or fan out after loss or an unresponsive -holder, and cancel it only after a matching content-validated header is pinned or the instance is -safely retired. +central integration actor keeps that timer active, initially fans out to two connected holders, +cycles through further holders every 250 ms after loss or an unresponsive response, and cancels the +request only after a matching content-validated header is pinned. The actor and its fetch state +outlive individual connection workers. Reconnection replays recipient-specific local INITs and +rematerializes locally authorized phase messages under the new connection. At least one honest holder exists in every `V`-stake READY set. At least `f + 1` honest holders exist in an equal-stake `2f + 1` ECHO quorum. Byzantine responses can delay retrieval but cannot change @@ -713,6 +715,13 @@ overhead number: saved INIT bytes can hide part of the phase-message cost. The b INIT/header bytes and ECHO/READY bytes separately; a one-tag lower-bound projection may be added but must be labeled as such. +The first integrated prototype also sends the canonical header once in RBC INIT and again inside +Starfish's existing full payload carrier. This deliberately preserves the transaction layer while +the certification boundary is validated, but it is avoidable duplicate traffic. Initial benchmark +results therefore measure the current whole-system prototype, not the minimum possible RBC header +overhead. The per-message framed-byte counters make this duplication and the ECHO/READY cost +visible separately. + Metrics should separate: - initial header-authentication bytes and CPU; @@ -763,26 +772,35 @@ Each milestone is committed separately. committee-bound pinned headers, context-bound ECHO capabilities, atomic poisoned-proof staging, bounded phase equivocation, round admission seams, pending triggers, and holder-backed multi-kernel recovery tests. The durable network fetch owner is part of milestone four. -4. **Certified Starfish integration:** add `starfish-rbc`, selectable initial authentication, the +4. **Certified Starfish integration (complete):** add `starfish-rbc`, selectable initial + authentication, the network RBC service and durable multi-holder fetch retry, dirty/clean lifecycle, clean-only - acknowledgments, and clean-only consensus/linearization. -5. **End-to-end validation:** poisoned-tag, equivocation, dangling-parent, and all-authentication - commit tests. + acknowledgments, clean-only consensus/linearization, per-peer ordered outbound isolation, and + fresh per-run protocol-instance distribution. +5. **End-to-end validation (in progress):** all four initial-authentication modes commit in a + four-validator network test, and a late-joining MAC validator catches up. Kernel-level + poisoned-tag and equivocation tests are complete; composed dangling-parent and Byzantine + network tests remain. 6. **Tree dissemination:** subtree tag bundles, redundant routing/fallback, and matching signature baselines. 7. **Recovery:** durable phase locks and delivered state, evidence replay, late-node synchronization, and restart tests. 8. **Benchmarks:** direct and tree comparisons with results reported outside this specification. -## 15. Remaining integration decisions +## 15. Remaining integration and research decisions The kernel behavior, content digest, authenticated encoding, size limits, and admission semantics -above are fixed. Integration must still choose: +above are fixed. The orchestrator generates one fresh nonzero protocol instance before serializing +the shared node configuration; version one uses two-holder recovery fanout with a 250 ms retry; and +the legacy unsafe aliases remain explicitly labeled lower bounds. Remaining work must choose: -- how benchmark genesis generates and distributes the fresh 32-byte `protocol_instance`; - a safe post-v1 state-retirement and garbage-collection rule; -- header-holder request fanout and retry timing; -- whether legacy unsafe `*-mac` aliases are renamed or retained as lower-bound benchmarks. +- a catch-up mechanism for an honest validator delayed by more than the current 100-round admission + window without reopening unbounded Byzantine slot allocation; +- a bounded per-peer RBC outbound queue policy that preserves honest-peer isolation under a stalled + receiver; +- removal of the duplicate header in the transaction payload carrier; and +- production-authenticated connection identity and durable phase/delivery recovery. The authentication selector remains `--block-authentication`; Starfish-RBC integration adds `mac` to the existing Ed25519, ML-DSA-44, and ML-DSA-65 values while retaining Ed25519 as the default. From c2ccf9ddcf6b7c2780f8b082f949eafea8a6ce85 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:13:13 +0200 Subject: [PATCH 25/62] Report benchmark traffic by message type --- crates/starfish-core/src/metrics.rs | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index b1de986e..15004578 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -1090,6 +1090,72 @@ impl Metrics { b->"Average bandwidth in:", format!("{:.2} MB/s", bw_in) ]); + const NETWORK_MESSAGE_TYPES: &[&str] = &[ + "subscribe_broadcast", + "batch", + "missing_parents", + "missing_tx_data", + "partial_sig", + "cert_echo", + "cert_vote", + "cert_ready", + "cert_batch", + "sailfish_timeout", + "sailfish_no_vote", + "unprovable_cert_request", + "round_gap_request", + "rbc_initial", + "rbc_echo", + "rbc_ready", + "rbc_header_request", + "rbc_header_response", + ]; + let outbound_message_breakdown = NETWORK_MESSAGE_TYPES + .iter() + .filter_map(|request_type| { + let average_bytes = metrics + .iter() + .map(|metrics| { + metrics + .network_message_bytes_sent_total + .with_label_values(&[request_type]) + .get() + }) + .sum::() as f64 + / num_validators as f64; + if average_bytes == 0.0 { + return None; + } + let average_requests = metrics + .iter() + .map(|metrics| { + metrics + .network_requests_sent_total + .with_label_values(&[request_type]) + .get() + }) + .sum::() as f64 + / num_validators as f64; + Some((*request_type, average_bytes, average_requests)) + }) + .collect::>(); + if !outbound_message_breakdown.is_empty() { + table.add_row(row![bH2->""]); + table.add_row(row![bH2->"Average Outbound Message Breakdown"]); + for (request_type, average_bytes, average_requests) in outbound_message_breakdown { + let bandwidth = average_bytes / duration_secs as f64 / 1024.0 / 1024.0; + let share = if average_bytes_sent == 0 { + 0.0 + } else { + average_bytes / average_bytes_sent as f64 * 100.0 + }; + let requests_per_second = average_requests / duration_secs as f64; + table.add_row(row![ + b->format!("{request_type}:"), + format!("{bandwidth:.3} MB/s ({share:.1}%, {requests_per_second:.1} msg/s)") + ]); + } + } let total_average_transactions = (average_tps * duration_secs as f64) as u64; let bandwidth_efficiency = if total_average_transactions > 0 { average_bytes_sent as f64 / total_average_transactions as f64 / 512.0 From 82bf18a575166aa72b402430a33f9d924a468aef Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:05:24 +0200 Subject: [PATCH 26/62] Send Starfish-RBC payloads without duplicate headers --- crates/starfish-core/src/broadcaster.rs | 33 ++- crates/starfish-core/src/dag_state.rs | 3 + crates/starfish-core/src/metrics.rs | 1 + crates/starfish-core/src/net_sync.rs | 194 ++++++++++++++- crates/starfish-core/src/network.rs | 51 +++- .../starfish-core/src/starfish_rbc_service.rs | 230 +++++++++++++++++- crates/starfish-core/src/syncer.rs | 5 +- docs/starfish-rbc-protocol.md | 36 +-- 8 files changed, 519 insertions(+), 34 deletions(-) diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index 13de59ed..a100f4ac 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -883,6 +883,11 @@ where *round = max(*round, block.round()); } } + if inner.dag_state.consensus_protocol.is_starfish_rbc() { + // INIT is the sole proactive header carrier and the RBC service sends + // transaction data as a separate reference-keyed payload. + return Some(()); + } tracing::debug!("Blocks to be sent to {peer} are {blocks:?}"); let batch = BlockBatch::full_only(DataSource::BlockBundleStreaming, blocks); if let Ok(size) = bincode::serialized_size(&batch) { @@ -1088,6 +1093,12 @@ where let useful_shards = AuthoritySet::default(); report_useful_authorities(metrics, peer.as_str(), useful_headers, useful_shards); + if inner.dag_state.consensus_protocol.is_starfish_rbc() { + let mut sent = sent_to_peer.write(); + sent.extend(blocks.iter().map(|block| *block.reference())); + return Some(()); + } + tracing::debug!("Blocks to be sent to {peer} are {blocks:?}"); let batch = BlockBatch { source: DataSource::BlockBundleStreaming, @@ -1356,9 +1367,15 @@ where } } PushOtherBlocksFormat::HeadersAndShards => { + let rbc_payload_sidecar = inner.dag_state.consensus_protocol.is_starfish_rbc(); + let header_refs = if rbc_payload_sidecar { + &[] + } else { + plan.other_refs.as_slice() + }; let (headers, shards) = inner .dag_state - .get_transmission_parts(&plan.other_refs, &plan.shard_refs); + .get_transmission_parts(header_refs, &plan.shard_refs); let headers = prepare_forwarded_blocks_for_peer( inner.dag_state.block_authentication_scheme, inner.dag_state.consensus_protocol, @@ -1367,7 +1384,11 @@ where ); BlockBatch { source: DataSource::BlockBundleStreaming, - full_blocks: plan.own_blocks, + full_blocks: if rbc_payload_sidecar { + Vec::new() + } else { + plan.own_blocks + }, headers, shards, useful_headers_authors: plan.useful_headers, @@ -1407,7 +1428,7 @@ where if let Some(max_round) = own_blocks.iter().map(|b| b.round()).max() { *round = max_round; } - if !own_blocks.is_empty() { + if !own_blocks.is_empty() && !inner.dag_state.consensus_protocol.is_starfish_rbc() { let fast_batch = BlockBatch::full_only(DataSource::BlockBundleStreaming, own_blocks.clone()); if let Ok(size) = bincode::serialized_size(&fast_batch) { @@ -1444,6 +1465,12 @@ where // Drop own blocks from the plan — already shipped in the fast batch. plan.own_blocks = Vec::new(); + if inner.dag_state.consensus_protocol.is_starfish_rbc() { + // RBC INIT/recovery owns header dissemination. The ordinary Starfish + // broadcaster remains responsible only for shard sidecars. + plan.other_refs.clear(); + plan.useful_headers = AuthoritySet::default(); + } let slow_batch = materialize_push_batch(&inner, to_whom_authority_index, plan); if slow_batch.is_empty() { diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index da4903ea..2941ad26 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -82,6 +82,8 @@ pub enum DataSource { /// Response to RoundGapRequest (blocks the requester was missing at a /// round). RoundGapResponse, + /// Header-free transaction data carried by Starfish-RBC after INIT. + StarfishRbcPayload, } impl DataSource { @@ -96,6 +98,7 @@ impl DataSource { Self::Recover => "recover", Self::UnprovableCertificateResponse => "unprovable_certificate_response", Self::RoundGapResponse => "round_gap_response", + Self::StarfishRbcPayload => "starfish_rbc_payload", } } } diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 15004578..9cb02ea6 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -1109,6 +1109,7 @@ impl Metrics { "rbc_ready", "rbc_header_request", "rbc_header_response", + "rbc_payload", ]; let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 829df318..8d841c35 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -38,21 +38,23 @@ use crate::{ dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, metrics::{Metrics, UtilizationTimerVecExt}, - network::{BlockBatch, Connection, Network, NetworkMessage, ShardPayload}, + network::{ + BlockBatch, Connection, Network, NetworkMessage, RbcTransactionPayload, ShardPayload, + }, runtime::{Handle, JoinError, JoinHandle, sleep}, sailfish_service::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, - starfish_rbc::RbcProtocolInstanceId, + starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, starfish_rbc_service::{ RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, }, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, - BlockReference, PartialSig, PartialSigKind, ProvableShard, RoundNumber, VerifiedBlock, - format_authority_index, + BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, + RoundNumber, VerifiedBlock, format_authority_index, }, }; @@ -103,6 +105,51 @@ fn verify_mac_transport( } } +fn verify_starfish_rbc_transaction_payload( + canonical_header: &RbcCanonicalHeader, + payload: RbcTransactionPayload, + committee: &Committee, + own_id: AuthorityIndex, + peer_id: AuthorityIndex, + encoder: &mut ReedSolomonEncoder, + authentication_scheme: BlockAuthenticationScheme, + mac_keys: &[MacKey], +) -> eyre::Result { + let (block_reference, transaction_data) = payload.into_parts(); + if block_reference != canonical_header.reference() { + eyre::bail!( + "Starfish-RBC transaction payload reference {} does not match header {}", + block_reference, + canonical_header.reference() + ); + } + let (block_header, _) = canonical_header.to_authentication_free_block().into_parts(); + let mut block = VerifiedBlock::from_parts(block_header, Some(transaction_data)); + let Some(mut shard_data) = block.verify_with_authentication( + committee, + own_id as usize, + peer_id as usize, + encoder, + ConsensusProtocol::StarfishRbc, + authentication_scheme, + mac_keys, + )? + else { + eyre::bail!("Starfish-RBC transaction payload for {block_reference} is empty"); + }; + block.preserialize(); + shard_data.preserialize(); + let transaction_data = block + .transaction_data() + .expect("verified RBC payload must contain transaction data") + .clone(); + Ok(ReconstructedTransactionData { + block_reference, + transaction_data, + shard_data, + }) +} + /// Prepare blocks forwarded through relay or synchronization paths for a /// specific peer. Legacy MAC-experiment blocks retain their complete vector /// only at direct recipients; forwarding selects the destination's tag. A @@ -945,6 +992,15 @@ impl ConnectionHandler { + if let Some(ref rbc) = self.starfish_rbc_service { + if let Err(error) = rbc.transaction_payload(self.peer_id, payload) { + tracing::warn!( + "Failed to forward Starfish-RBC transaction payload: {error}" + ); + } + } + } } true } @@ -1743,6 +1799,8 @@ impl NetworkSyncer let rbc_event_task = rbc_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); handle.spawn(async move { + let mut payload_encoder = ReedSolomonEncoder::new(2, 4, 2) + .expect("Starfish-RBC payload encoder should be created"); while let Some(event) = event_rx.recv().await { match event { RbcServiceEvent::Network { recipient, message } => { @@ -1789,6 +1847,47 @@ impl NetworkSyncer ); } } + RbcServiceEvent::TransactionPayloadStaged { + peer, + header, + payload, + } => { + let block_ref = payload.block_reference(); + let item = match verify_starfish_rbc_transaction_payload( + header.header(), + payload, + &event_inner.committee, + event_inner.dag_state.get_own_authority_index(), + peer, + &mut payload_encoder, + event_inner.dag_state.block_authentication_scheme, + &event_inner.mac_keys, + ) { + Ok(item) => item, + Err(error) => { + tracing::warn!( + ?block_ref, + peer, + ?error, + "Rejected Starfish-RBC transaction payload" + ); + continue; + } + }; + event_inner + .cordial_knowledge + .send(CordialKnowledgeMessage::DagParts { + headers: Vec::new(), + shards: vec![block_ref], + }); + if let Some(shard_tx) = event_inner.shard_tx.lock().as_ref() { + let _ = shard_tx.send(vec![ShardMessage::FullBlock(block_ref)]); + } + event_inner + .syncer + .add_transaction_data(vec![item], DataSource::StarfishRbcPayload) + .await; + } RbcServiceEvent::Delivered(header) => { event_inner .syncer @@ -2750,8 +2849,9 @@ mod tests { use super::*; use crate::{ - crypto::{self, SignatureBytes}, - types::{BaseTransaction, BlockReference}, + crypto::{self, SignatureBytes, TransactionsCommitment}, + encoder::ShardEncoder, + types::{BaseTransaction, BlockReference, Transaction, TransactionData}, }; #[tokio::test] @@ -2768,6 +2868,88 @@ mod tests { wait.await; } + #[test] + fn starfish_rbc_payload_is_header_free_and_commitment_checked() { + let committee = Committee::new_test(vec![1; 4]); + let transactions = vec![BaseTransaction::Share(Transaction::new(vec![7; 64]))]; + let mut commitment_encoder = + ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); + let encoded = commitment_encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + let (commitment, _) = TransactionsCommitment::new_from_encoded_transactions(&encoded, 1); + let canonical = RbcCanonicalHeader::try_new( + 0, + 1, + vec![ + BlockReference::new_test(0, 0), + BlockReference::new_test(1, 0), + BlockReference::new_test(2, 0), + ], + Vec::new(), + 11, + commitment, + ) + .unwrap(); + let payload = RbcTransactionPayload::new( + canonical.reference(), + TransactionData::new(transactions.clone()), + ); + let mut verifier = ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); + let verified = verify_starfish_rbc_transaction_payload( + &canonical, + payload, + &committee, + 1, + 0, + &mut verifier, + BlockAuthenticationScheme::MacVector, + &[], + ) + .unwrap(); + assert_eq!(verified.block_reference, canonical.reference()); + assert_eq!(verified.transaction_data.transactions(), &transactions); + assert_eq!(verified.shard_data.shard_index(), 1); + + let tampered = RbcTransactionPayload::new( + canonical.reference(), + TransactionData::new(vec![BaseTransaction::Share(Transaction::new(vec![8; 64]))]), + ); + assert!( + verify_starfish_rbc_transaction_payload( + &canonical, + tampered, + &committee, + 1, + 0, + &mut verifier, + BlockAuthenticationScheme::MacVector, + &[], + ) + .is_err() + ); + + let wrong_reference = RbcTransactionPayload::new( + BlockReference::new_test(0, 2), + TransactionData::new(transactions), + ); + assert!( + verify_starfish_rbc_transaction_payload( + &canonical, + wrong_reference, + &committee, + 1, + 0, + &mut verifier, + BlockAuthenticationScheme::MacVector, + &[], + ) + .is_err() + ); + } + #[test] fn block_filter_allows_exactly_one_tag_to_full_mac_upgrade() { let filter = FilterForBlocks::new(); diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 5ee30976..0f682838 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -32,7 +32,8 @@ use crate::{ stat::HistogramSender, types::{ AuthorityIndex, AuthoritySet, BlockReference, CertMessage, CertMessageKind, PartialSig, - ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, VerifiedBlock, + ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, TransactionData, + VerifiedBlock, }, }; @@ -83,6 +84,44 @@ pub struct ShardPayload { pub shard: ProvableShard, } +/// Starfish-RBC transaction data transported separately from the canonical +/// header already carried by INIT or header recovery. +#[derive(Clone, Serialize, Deserialize)] +pub struct RbcTransactionPayload { + block_reference: BlockReference, + transaction_data: TransactionData, +} + +impl RbcTransactionPayload { + pub(crate) fn new(block_reference: BlockReference, transaction_data: TransactionData) -> Self { + Self { + block_reference, + transaction_data, + } + } + + pub(crate) fn block_reference(&self) -> BlockReference { + self.block_reference + } + + pub(crate) fn into_parts(self) -> (BlockReference, TransactionData) { + (self.block_reference, self.transaction_data) + } +} + +impl std::fmt::Debug for RbcTransactionPayload { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RbcTransactionPayload") + .field("block_reference", &self.block_reference) + .field( + "transaction_count", + &self.transaction_data.number_transactions(), + ) + .finish() + } +} + /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -190,6 +229,9 @@ pub enum NetworkMessage { /// Starfish-RBC: return canonical header content. The receiver recomputes /// and checks its content-addressed reference before accepting it. RbcHeaderResponse(RbcCanonicalHeader), + /// Starfish-RBC: transaction data keyed by the canonical reference already + /// disseminated through INIT. This deliberately carries no block header. + RbcTransactionPayload(RbcTransactionPayload), } impl NetworkMessage { @@ -217,6 +259,7 @@ impl NetworkMessage { }, Self::RbcHeaderRequest(_) => "rbc_header_request", Self::RbcHeaderResponse(_) => "rbc_header_response", + Self::RbcTransactionPayload(_) => "rbc_payload", } } } @@ -966,6 +1009,7 @@ mod tests { use crate::{ crypto::{MacTag, TransactionsCommitment, dummy_signer}, starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, + types::TransactionData, }; fn variant_index(message: &NetworkMessage) -> u32 { @@ -1006,12 +1050,17 @@ mod tests { )); let request = NetworkMessage::RbcHeaderRequest(block_ref); let response = NetworkMessage::RbcHeaderResponse(header); + let payload = NetworkMessage::RbcTransactionPayload(RbcTransactionPayload::new( + block_ref, + TransactionData::new(Vec::new()), + )); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), (phase, 12, "rbc_ready"), (request, 13, "rbc_header_request"), (response, 14, "rbc_header_response"), + (payload, 15, "rbc_payload"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 3962fb1b..417a1e38 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -26,7 +26,7 @@ use tokio::{ use crate::{ committee::Committee, crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, - network::NetworkMessage, + network::{NetworkMessage, RbcTransactionPayload}, starfish_rbc::{ PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, RbcInitialHeaderOutcome, RbcInitialProof, RbcLocalInitial, RbcPhase, RbcPhaseMessage, @@ -34,7 +34,7 @@ use crate::{ }, types::{ AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - RoundNumber, TimestampNs, + RoundNumber, TimestampNs, TransactionData, }, }; @@ -97,6 +97,11 @@ pub(crate) enum RbcServiceError { block_ref: BlockReference, peer: AuthorityIndex, }, + UnexpectedTransactionPayload(BlockReference), + TransactionPayloadNotFromAuthor { + block_ref: BlockReference, + peer: AuthorityIndex, + }, ServiceStopped, } @@ -133,6 +138,14 @@ impl fmt::Display for RbcServiceError { formatter, "Starfish-RBC header response for {block_ref} came from non-holder {peer}" ), + Self::UnexpectedTransactionPayload(block_ref) => write!( + formatter, + "Starfish-RBC transaction payload arrived before its header for {block_ref}" + ), + Self::TransactionPayloadNotFromAuthor { block_ref, peer } => write!( + formatter, + "Starfish-RBC transaction payload for {block_ref} came from non-author {peer}" + ), Self::ServiceStopped => formatter.write_str("Starfish-RBC service stopped"), } } @@ -148,6 +161,11 @@ pub(crate) enum RbcServiceEvent { message: NetworkMessage, }, HeaderStaged(PinnedRbcHeader), + TransactionPayloadStaged { + peer: AuthorityIndex, + header: PinnedRbcHeader, + payload: RbcTransactionPayload, + }, Delivered(PinnedRbcHeader), Rejected { peer: Option, @@ -158,6 +176,7 @@ pub(crate) enum RbcServiceEvent { enum RbcServiceMessage { StartLocal { header: RbcLocalHeader, + transaction_data: Option, reply: oneshot::Sender>, }, DirectInitial { @@ -176,6 +195,10 @@ enum RbcServiceMessage { peer: AuthorityIndex, header: RbcCanonicalHeader, }, + TransactionPayload { + peer: AuthorityIndex, + payload: RbcTransactionPayload, + }, PeerConnected(AuthorityIndex), PeerDisconnected(AuthorityIndex), #[allow(dead_code)] @@ -197,9 +220,22 @@ impl RbcServiceHandle { pub(crate) async fn start_local_header( &self, header: RbcLocalHeader, + ) -> Result { + self.start_local_header_with_payload(header, None).await + } + + #[allow(dead_code)] + pub(crate) async fn start_local_header_with_payload( + &self, + header: RbcLocalHeader, + transaction_data: Option, ) -> Result { let (reply, receiver) = oneshot::channel(); - self.send(RbcServiceMessage::StartLocal { header, reply })?; + self.send(RbcServiceMessage::StartLocal { + header, + transaction_data, + reply, + })?; receiver .await .map_err(|_| RbcServiceError::ServiceStopped)? @@ -209,12 +245,17 @@ impl RbcServiceHandle { /// completed only after the kernel has selected/pinned the slot and all /// INIT/phase events have been enqueued, so legacy dissemination cannot /// race ahead of local RBC authorization. - pub(crate) fn start_local_header_blocking( + pub(crate) fn start_local_header_with_payload_blocking( &self, header: RbcLocalHeader, + transaction_data: Option, ) -> Result { let (reply, receiver) = oneshot::channel(); - self.send(RbcServiceMessage::StartLocal { header, reply })?; + self.send(RbcServiceMessage::StartLocal { + header, + transaction_data, + reply, + })?; receiver .blocking_recv() .map_err(|_| RbcServiceError::ServiceStopped)? @@ -252,6 +293,14 @@ impl RbcServiceHandle { self.send(RbcServiceMessage::HeaderResponse { peer, header }) } + pub(crate) fn transaction_payload( + &self, + peer: AuthorityIndex, + payload: RbcTransactionPayload, + ) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::TransactionPayload { peer, payload }) + } + pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), RbcServiceError> { self.send(RbcServiceMessage::PeerConnected(peer)) } @@ -339,6 +388,7 @@ pub(crate) fn start_starfish_rbc_service( pending_fetches: AHashMap::new(), staged_notifications: AHashSet::new(), retained_initials: BTreeMap::new(), + retained_transaction_payloads: BTreeMap::new(), retained_phases: BTreeSet::new(), }; let task = tokio::spawn(run_service(state, message_rx, header_retry_interval)); @@ -410,6 +460,10 @@ struct RbcServiceState { /// Recipient-specialized local proposals retained for replay after a /// connection is replaced. Version one keeps these for the run. retained_initials: BTreeMap<(BlockReference, AuthorityIndex), RbcHeaderProposal>, + /// Header-free transaction payloads retained for replay after INIT. The + /// same content-addressed payload is specialized only by its recipient + /// routing, not by its bytes. + retained_transaction_payloads: BTreeMap, /// Authorized local phase intents. Tags are rematerialized for the peer /// on replay rather than retaining or cloning a tagged wire message. retained_phases: BTreeSet<(BlockReference, RbcPhase)>, @@ -418,8 +472,12 @@ struct RbcServiceState { impl RbcServiceState { fn process_message(&mut self, message: RbcServiceMessage) { match message { - RbcServiceMessage::StartLocal { header, reply } => { - let result = self.start_local_header(header); + RbcServiceMessage::StartLocal { + header, + transaction_data, + reply, + } => { + let result = self.start_local_header(header, transaction_data); let _ = reply.send(result); } RbcServiceMessage::DirectInitial { peer, proposal } => { @@ -437,6 +495,9 @@ impl RbcServiceState { RbcServiceMessage::HeaderResponse { peer, header } => { self.accept_header_response(peer, header); } + RbcServiceMessage::TransactionPayload { peer, payload } => { + self.accept_transaction_payload(peer, payload); + } RbcServiceMessage::PeerConnected(peer) => self.peer_connected(peer), RbcServiceMessage::PeerDisconnected(peer) => self.peer_disconnected(peer), RbcServiceMessage::AdvanceLocalRound { round, reply } => { @@ -456,6 +517,7 @@ impl RbcServiceState { fn start_local_header( &mut self, header: RbcLocalHeader, + transaction_data: Option, ) -> Result { self.kernel.advance_local_round(header.round)?; let local = self.kernel.start_local_initial_header( @@ -468,12 +530,24 @@ impl RbcServiceState { let canonical = local.header().clone(); let proposals = self.make_initial_proposals(&local); let (pinned, effects) = local.into_parts(); + let transaction_payload = + transaction_data.map(|data| RbcTransactionPayload::new(canonical.reference(), data)); + if let Some(payload) = transaction_payload.as_ref() { + self.retained_transaction_payloads + .insert(canonical.reference(), payload.clone()); + } self.notify_header_staged(pinned); for (recipient, proposal) in proposals { self.retained_initials .insert((canonical.reference(), recipient), proposal.clone()); self.send_network(recipient, NetworkMessage::RbcInitial(proposal)); + if let Some(payload) = transaction_payload.as_ref() { + self.send_network( + recipient, + NetworkMessage::RbcTransactionPayload(payload.clone()), + ); + } } self.process_effects(effects); Ok(canonical) @@ -619,6 +693,35 @@ impl RbcServiceState { } } + fn accept_transaction_payload(&mut self, peer: AuthorityIndex, payload: RbcTransactionPayload) { + let block_ref = payload.block_reference(); + if !self.committee.known_authority(peer) { + self.reject(Some(peer), RbcError::UnknownAuthority(peer).into()); + return; + } + if peer != block_ref.authority { + self.reject( + Some(peer), + RbcServiceError::TransactionPayloadNotFromAuthor { block_ref, peer }, + ); + return; + } + match self.kernel.pinned_header(block_ref) { + Ok(Some(header)) => { + let _ = self.events.send(RbcServiceEvent::TransactionPayloadStaged { + peer, + header, + payload, + }); + } + Ok(None) => self.reject( + Some(peer), + RbcServiceError::UnexpectedTransactionPayload(block_ref), + ), + Err(error) => self.reject(Some(peer), error.into()), + } + } + fn finish_header_staging(&mut self, block_ref: BlockReference, peer: Option) { match self.kernel.pinned_header(block_ref) { Ok(Some(header)) => { @@ -748,6 +851,15 @@ impl RbcServiceState { self.send_network(peer, NetworkMessage::RbcInitial(proposal)); } + let payloads: Vec<_> = self + .retained_transaction_payloads + .values() + .cloned() + .collect(); + for payload in payloads { + self.send_network(peer, NetworkMessage::RbcTransactionPayload(payload)); + } + let phases: Vec<_> = self.retained_phases.iter().copied().collect(); for (block_ref, phase) in phases { match self.kernel.make_phase_message(phase, block_ref, peer) { @@ -802,7 +914,7 @@ mod tests { dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, starfish_rbc::RbcPhase, - types::VerifiedBlock, + types::{TransactionData, VerifiedBlock}, }; fn instance() -> RbcProtocolInstanceId { @@ -933,7 +1045,7 @@ mod tests { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); let blocking_handle = handle.clone(); let canonical = tokio::task::spawn_blocking(move || { - blocking_handle.start_local_header_blocking(local_header(1, 4)) + blocking_handle.start_local_header_with_payload_blocking(local_header(1, 4), None) }) .await .unwrap() @@ -948,6 +1060,106 @@ mod tests { task.await.unwrap(); } + #[tokio::test] + async fn local_payload_is_header_free_and_replayed_after_initial() { + let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::MacVector); + let transaction_data = TransactionData::new(Vec::new()); + let canonical = handle + .start_local_header_with_payload(local_header(1, 4), Some(transaction_data)) + .await + .unwrap(); + + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::HeaderStaged(ref header) if header.reference() == canonical.reference() + )); + for expected_recipient in 1..4 { + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient, + message: NetworkMessage::RbcInitial(ref proposal), + } if recipient == expected_recipient && proposal.header() == &canonical + )); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient, + message: NetworkMessage::RbcTransactionPayload(ref payload), + } if recipient == expected_recipient + && payload.block_reference() == canonical.reference() + )); + } + + // Drain the three initial ECHO messages, then reconnect one peer. The + // replay FIFO must put INIT and payload before the rematerialized ECHO. + for _ in 0..3 { + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + message: NetworkMessage::RbcPhase(_), + .. + } + )); + } + handle.peer_connected(2).unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient: 2, + message: NetworkMessage::RbcInitial(_), + } + )); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient: 2, + message: NetworkMessage::RbcTransactionPayload(ref payload), + } if payload.block_reference() == canonical.reference() + )); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Network { + recipient: 2, + message: NetworkMessage::RbcPhase(ref message), + } if message.phase() == RbcPhase::Echo + )); + + drop(handle); + task.await.unwrap(); + } + + #[tokio::test] + async fn transaction_payload_requires_its_pinned_header_and_direct_author() { + let (handle, mut events, task) = start_service(1, BlockAuthenticationScheme::MacVector); + let block_ref = BlockReference::new_test(0, 1); + let payload = RbcTransactionPayload::new(block_ref, TransactionData::new(Vec::new())); + + handle.transaction_payload(0, payload.clone()).unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Rejected { + peer: Some(0), + error: RbcServiceError::UnexpectedTransactionPayload(reference), + } if reference == block_ref + )); + + handle.transaction_payload(2, payload).unwrap(); + assert!(matches!( + next_event(&mut events).await, + RbcServiceEvent::Rejected { + peer: Some(2), + error: RbcServiceError::TransactionPayloadNotFromAuthor { + block_ref: reference, + peer: 2, + }, + } if reference == block_ref + )); + + drop(handle); + task.await.unwrap(); + } + #[tokio::test] async fn local_start_advances_the_admission_window_before_pinning() { let (handle, events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 6892fc9c..c0652e6e 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -344,7 +344,10 @@ impl Syncer { .starfish_rbc_service .as_ref() .expect("Starfish-RBC protocol must start its RBC service") - .start_local_header_blocking(RbcLocalHeader::from_canonical(&canonical)) + .start_local_header_with_payload_blocking( + RbcLocalHeader::from_canonical(&canonical), + block.transaction_data().cloned(), + ) .expect("local Starfish-RBC header must be accepted before dissemination"); assert_eq!( selected.reference(), diff --git a/docs/starfish-rbc-protocol.md b/docs/starfish-rbc-protocol.md index 72a7ac03..bef38115 100644 --- a/docs/starfish-rbc-protocol.md +++ b/docs/starfish-rbc-protocol.md @@ -278,6 +278,11 @@ HeaderResponse { slot, canonical_header, } + +TransactionPayload { + block_ref, + transaction_data, +} ``` The protocol instance, committee ID, and authentication mode are fixed service context and need not @@ -285,9 +290,9 @@ be repeated on the wire, but every tag authenticates them. ECHO and READY contai not the header. This avoids rebroadcasting each header quadratically. Header request/response traffic transports data only and is never counted as quorum testimony. -The milestone-two phase message has a golden bincode regression vector. The eventual -`NetworkMessage` integration must append a new variant or use a versioned envelope; it must not -silently change existing variant discriminants. +The milestone-two phase message has a golden bincode regression vector. Integrated +`NetworkMessage` variants are append-only so they do not silently change existing bincode +discriminants. An honest ECHO or READY sender must possess the matching content-validated header. Consequently, when a threshold is observed before the local header arrives, the receiver can request the header @@ -298,8 +303,13 @@ content digest and validating the header's structure. needs nor confers a valid local initial proof. It supplies the content bytes needed for a transition that is authorized by the receiver's own directly authenticated RBC evidence. -The RBC header-retrieval path never asserts transaction-payload availability. Shards and full -transaction data continue to use the existing Starfish paths. +The RBC header-retrieval path never asserts transaction-payload availability. Direct transaction +data uses a header-free payload keyed by the already authenticated `BlockReference`; the receiver +checks it against the transaction commitment in the pinned header, derives its local shard, and +attaches or buffers it through the existing Starfish transaction-data path. Shard relay and +reconstruction remain unchanged. Explicit missing-parent synchronization may still return a +header as a recovery fallback, but proactive full-block/header batches are disabled for +Starfish-RBC. ## 5. Local state @@ -715,12 +725,11 @@ overhead number: saved INIT bytes can hide part of the phase-message cost. The b INIT/header bytes and ECHO/READY bytes separately; a one-tag lower-bound projection may be added but must be labeled as such. -The first integrated prototype also sends the canonical header once in RBC INIT and again inside -Starfish's existing full payload carrier. This deliberately preserves the transaction layer while -the certification boundary is validated, but it is avoidable duplicate traffic. Initial benchmark -results therefore measure the current whole-system prototype, not the minimum possible RBC header -overhead. The per-message framed-byte counters make this duplication and the ECHO/READY cost -visible separately. +Starfish-RBC sends the canonical header through INIT (or explicit header recovery) and transports +direct transaction data separately as `RbcTransactionPayload { block_ref, transaction_data }`. +This removes the steady-state duplicate header that the first integrated prototype carried inside +an ordinary full-block batch. The per-message framed-byte counters report the payload and +ECHO/READY costs separately. Metrics should separate: @@ -775,8 +784,8 @@ Each milestone is committed separately. 4. **Certified Starfish integration (complete):** add `starfish-rbc`, selectable initial authentication, the network RBC service and durable multi-holder fetch retry, dirty/clean lifecycle, clean-only - acknowledgments, clean-only consensus/linearization, per-peer ordered outbound isolation, and - fresh per-run protocol-instance distribution. + acknowledgments, clean-only consensus/linearization, per-peer ordered outbound isolation, + header-free transaction payloads, and fresh per-run protocol-instance distribution. 5. **End-to-end validation (in progress):** all four initial-authentication modes commit in a four-validator network test, and a late-joining MAC validator catches up. Kernel-level poisoned-tag and equivocation tests are complete; composed dangling-parent and Byzantine @@ -799,7 +808,6 @@ the legacy unsafe aliases remain explicitly labeled lower bounds. Remaining work window without reopening unbounded Byzantine slot allocation; - a bounded per-peer RBC outbound queue policy that preserves honest-peer isolation under a stalled receiver; -- removal of the duplicate header in the transaction payload carrier; and - production-authenticated connection identity and durable phase/delivery recovery. The authentication selector remains `--block-authentication`; Starfish-RBC integration adds `mac` From 58b4475d086d52512b018042c08e8ac87cbf5cb5 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:22:07 +0200 Subject: [PATCH 27/62] Co-locate Starfish-RBC payloads with INIT --- Cargo.toml | 2 +- crates/starfish-core/src/broadcaster.rs | 4 +- crates/starfish-core/src/dag_state.rs | 2 +- crates/starfish-core/src/metrics.rs | 1 - crates/starfish-core/src/net_sync.rs | 65 +---- crates/starfish-core/src/network.rs | 51 +--- crates/starfish-core/src/starfish_rbc.rs | 57 ++++- .../starfish-core/src/starfish_rbc_service.rs | 242 ++++++++---------- crates/starfish-core/src/types.rs | 9 + docs/starfish-rbc-protocol.md | 31 +-- 10 files changed, 198 insertions(+), 266 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9efcc775..1f7c4821 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ color-eyre = "0.6.2" eyre = "0.6.8" futures = "0.3.28" reqwest = { version = "0.12", features = ["json"] } -serde = { version = "1.0.163", features = ["derive"] } +serde = { version = "1.0.163", features = ["derive", "rc"] } tempfile = "3.6.0" tokio = { version = "1.28.1", features = ["full"] } tracing = "0.1.37" diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index a100f4ac..881331f4 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -884,8 +884,8 @@ where } } if inner.dag_state.consensus_protocol.is_starfish_rbc() { - // INIT is the sole proactive header carrier and the RBC service sends - // transaction data as a separate reference-keyed payload. + // INIT is the sole proactive header carrier and co-carries direct + // transaction data without repeating the header. return Some(()); } tracing::debug!("Blocks to be sent to {peer} are {blocks:?}"); diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 2941ad26..8e008bb0 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -82,7 +82,7 @@ pub enum DataSource { /// Response to RoundGapRequest (blocks the requester was missing at a /// round). RoundGapResponse, - /// Header-free transaction data carried by Starfish-RBC after INIT. + /// Transaction data co-carried by the direct Starfish-RBC INIT. StarfishRbcPayload, } diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 9cb02ea6..15004578 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -1109,7 +1109,6 @@ impl Metrics { "rbc_ready", "rbc_header_request", "rbc_header_response", - "rbc_payload", ]; let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 8d841c35..75adafec 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -38,9 +38,7 @@ use crate::{ dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, metrics::{Metrics, UtilizationTimerVecExt}, - network::{ - BlockBatch, Connection, Network, NetworkMessage, RbcTransactionPayload, ShardPayload, - }, + network::{BlockBatch, Connection, Network, NetworkMessage, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, sailfish_service::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, @@ -54,7 +52,7 @@ use crate::{ types::{ AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, - RoundNumber, VerifiedBlock, format_authority_index, + RoundNumber, TransactionData, VerifiedBlock, format_authority_index, }, }; @@ -107,7 +105,7 @@ fn verify_mac_transport( fn verify_starfish_rbc_transaction_payload( canonical_header: &RbcCanonicalHeader, - payload: RbcTransactionPayload, + transaction_data: Arc, committee: &Committee, own_id: AuthorityIndex, peer_id: AuthorityIndex, @@ -115,14 +113,8 @@ fn verify_starfish_rbc_transaction_payload( authentication_scheme: BlockAuthenticationScheme, mac_keys: &[MacKey], ) -> eyre::Result { - let (block_reference, transaction_data) = payload.into_parts(); - if block_reference != canonical_header.reference() { - eyre::bail!( - "Starfish-RBC transaction payload reference {} does not match header {}", - block_reference, - canonical_header.reference() - ); - } + let block_reference = canonical_header.reference(); + let transaction_data = Arc::try_unwrap(transaction_data).unwrap_or_else(|data| (*data).clone()); let (block_header, _) = canonical_header.to_authentication_free_block().into_parts(); let mut block = VerifiedBlock::from_parts(block_header, Some(transaction_data)); let Some(mut shard_data) = block.verify_with_authentication( @@ -992,15 +984,6 @@ impl ConnectionHandler { - if let Some(ref rbc) = self.starfish_rbc_service { - if let Err(error) = rbc.transaction_payload(self.peer_id, payload) { - tracing::warn!( - "Failed to forward Starfish-RBC transaction payload: {error}" - ); - } - } - } } true } @@ -1850,12 +1833,12 @@ impl NetworkSyncer RbcServiceEvent::TransactionPayloadStaged { peer, header, - payload, + transaction_data, } => { - let block_ref = payload.block_reference(); + let block_ref = header.reference(); let item = match verify_starfish_rbc_transaction_payload( header.header(), - payload, + transaction_data, &event_inner.committee, event_inner.dag_state.get_own_authority_index(), peer, @@ -2869,7 +2852,7 @@ mod tests { } #[test] - fn starfish_rbc_payload_is_header_free_and_commitment_checked() { + fn starfish_rbc_initial_payload_is_commitment_checked() { let committee = Committee::new_test(vec![1; 4]); let transactions = vec![BaseTransaction::Share(Transaction::new(vec![7; 64]))]; let mut commitment_encoder = @@ -2893,10 +2876,7 @@ mod tests { commitment, ) .unwrap(); - let payload = RbcTransactionPayload::new( - canonical.reference(), - TransactionData::new(transactions.clone()), - ); + let payload = Arc::new(TransactionData::new(transactions.clone())); let mut verifier = ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); let verified = verify_starfish_rbc_transaction_payload( &canonical, @@ -2913,10 +2893,9 @@ mod tests { assert_eq!(verified.transaction_data.transactions(), &transactions); assert_eq!(verified.shard_data.shard_index(), 1); - let tampered = RbcTransactionPayload::new( - canonical.reference(), - TransactionData::new(vec![BaseTransaction::Share(Transaction::new(vec![8; 64]))]), - ); + let tampered = Arc::new(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![8; 64]), + )])); assert!( verify_starfish_rbc_transaction_payload( &canonical, @@ -2930,24 +2909,6 @@ mod tests { ) .is_err() ); - - let wrong_reference = RbcTransactionPayload::new( - BlockReference::new_test(0, 2), - TransactionData::new(transactions), - ); - assert!( - verify_starfish_rbc_transaction_payload( - &canonical, - wrong_reference, - &committee, - 1, - 0, - &mut verifier, - BlockAuthenticationScheme::MacVector, - &[], - ) - .is_err() - ); } #[test] diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 0f682838..5ee30976 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -32,8 +32,7 @@ use crate::{ stat::HistogramSender, types::{ AuthorityIndex, AuthoritySet, BlockReference, CertMessage, CertMessageKind, PartialSig, - ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, TransactionData, - VerifiedBlock, + ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, VerifiedBlock, }, }; @@ -84,44 +83,6 @@ pub struct ShardPayload { pub shard: ProvableShard, } -/// Starfish-RBC transaction data transported separately from the canonical -/// header already carried by INIT or header recovery. -#[derive(Clone, Serialize, Deserialize)] -pub struct RbcTransactionPayload { - block_reference: BlockReference, - transaction_data: TransactionData, -} - -impl RbcTransactionPayload { - pub(crate) fn new(block_reference: BlockReference, transaction_data: TransactionData) -> Self { - Self { - block_reference, - transaction_data, - } - } - - pub(crate) fn block_reference(&self) -> BlockReference { - self.block_reference - } - - pub(crate) fn into_parts(self) -> (BlockReference, TransactionData) { - (self.block_reference, self.transaction_data) - } -} - -impl std::fmt::Debug for RbcTransactionPayload { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RbcTransactionPayload") - .field("block_reference", &self.block_reference) - .field( - "transaction_count", - &self.transaction_data.number_transactions(), - ) - .finish() - } -} - /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -229,9 +190,6 @@ pub enum NetworkMessage { /// Starfish-RBC: return canonical header content. The receiver recomputes /// and checks its content-addressed reference before accepting it. RbcHeaderResponse(RbcCanonicalHeader), - /// Starfish-RBC: transaction data keyed by the canonical reference already - /// disseminated through INIT. This deliberately carries no block header. - RbcTransactionPayload(RbcTransactionPayload), } impl NetworkMessage { @@ -259,7 +217,6 @@ impl NetworkMessage { }, Self::RbcHeaderRequest(_) => "rbc_header_request", Self::RbcHeaderResponse(_) => "rbc_header_response", - Self::RbcTransactionPayload(_) => "rbc_payload", } } } @@ -1009,7 +966,6 @@ mod tests { use crate::{ crypto::{MacTag, TransactionsCommitment, dummy_signer}, starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, - types::TransactionData, }; fn variant_index(message: &NetworkMessage) -> u32 { @@ -1050,17 +1006,12 @@ mod tests { )); let request = NetworkMessage::RbcHeaderRequest(block_ref); let response = NetworkMessage::RbcHeaderResponse(header); - let payload = NetworkMessage::RbcTransactionPayload(RbcTransactionPayload::new( - block_ref, - TransactionData::new(Vec::new()), - )); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), (phase, 12, "rbc_ready"), (request, 13, "rbc_header_request"), (response, 14, "rbc_header_response"), - (payload, 15, "rbc_payload"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index 6befce63..684f0995 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -21,7 +21,8 @@ use crate::{ types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - TimestampNs, VerifiedBlock, compress_acknowledgments, expand_acknowledgments, + TimestampNs, TransactionData, VerifiedBlock, compress_acknowledgments, + expand_acknowledgments, }, }; @@ -491,15 +492,33 @@ pub enum RbcInitialProof { /// /// The proof is a sidecar over the canonical header reference. It is not part /// of the content-addressed header identity. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct RbcHeaderProposal { header: RbcCanonicalHeader, proof: RbcInitialProof, + transaction_data: Option>, } impl RbcHeaderProposal { + #[cfg(test)] pub(crate) fn new(header: RbcCanonicalHeader, proof: RbcInitialProof) -> Self { - Self { header, proof } + Self { + header, + proof, + transaction_data: None, + } + } + + pub(crate) fn with_transaction_data( + header: RbcCanonicalHeader, + proof: RbcInitialProof, + transaction_data: Option>, + ) -> Self { + Self { + header, + proof, + transaction_data, + } } pub fn header(&self) -> &RbcCanonicalHeader { @@ -510,8 +529,36 @@ impl RbcHeaderProposal { &self.proof } - pub(crate) fn into_parts(self) -> (RbcCanonicalHeader, RbcInitialProof) { - (self.header, self.proof) + #[cfg(test)] + pub(crate) fn transaction_data(&self) -> Option<&TransactionData> { + self.transaction_data.as_deref() + } + + pub(crate) fn into_parts( + self, + ) -> ( + RbcCanonicalHeader, + RbcInitialProof, + Option>, + ) { + (self.header, self.proof, self.transaction_data) + } +} + +impl fmt::Debug for RbcHeaderProposal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RbcHeaderProposal") + .field("header", &self.header) + .field("proof", &self.proof) + .field( + "transaction_count", + &self + .transaction_data + .as_ref() + .map(|data| data.number_transactions()), + ) + .finish() } } diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 417a1e38..30dbf87b 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -26,7 +26,7 @@ use tokio::{ use crate::{ committee::Committee, crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, - network::{NetworkMessage, RbcTransactionPayload}, + network::NetworkMessage, starfish_rbc::{ PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, RbcInitialHeaderOutcome, RbcInitialProof, RbcLocalInitial, RbcPhase, RbcPhaseMessage, @@ -97,11 +97,6 @@ pub(crate) enum RbcServiceError { block_ref: BlockReference, peer: AuthorityIndex, }, - UnexpectedTransactionPayload(BlockReference), - TransactionPayloadNotFromAuthor { - block_ref: BlockReference, - peer: AuthorityIndex, - }, ServiceStopped, } @@ -138,14 +133,6 @@ impl fmt::Display for RbcServiceError { formatter, "Starfish-RBC header response for {block_ref} came from non-holder {peer}" ), - Self::UnexpectedTransactionPayload(block_ref) => write!( - formatter, - "Starfish-RBC transaction payload arrived before its header for {block_ref}" - ), - Self::TransactionPayloadNotFromAuthor { block_ref, peer } => write!( - formatter, - "Starfish-RBC transaction payload for {block_ref} came from non-author {peer}" - ), Self::ServiceStopped => formatter.write_str("Starfish-RBC service stopped"), } } @@ -164,7 +151,7 @@ pub(crate) enum RbcServiceEvent { TransactionPayloadStaged { peer: AuthorityIndex, header: PinnedRbcHeader, - payload: RbcTransactionPayload, + transaction_data: Arc, }, Delivered(PinnedRbcHeader), Rejected { @@ -195,10 +182,6 @@ enum RbcServiceMessage { peer: AuthorityIndex, header: RbcCanonicalHeader, }, - TransactionPayload { - peer: AuthorityIndex, - payload: RbcTransactionPayload, - }, PeerConnected(AuthorityIndex), PeerDisconnected(AuthorityIndex), #[allow(dead_code)] @@ -293,14 +276,6 @@ impl RbcServiceHandle { self.send(RbcServiceMessage::HeaderResponse { peer, header }) } - pub(crate) fn transaction_payload( - &self, - peer: AuthorityIndex, - payload: RbcTransactionPayload, - ) -> Result<(), RbcServiceError> { - self.send(RbcServiceMessage::TransactionPayload { peer, payload }) - } - pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), RbcServiceError> { self.send(RbcServiceMessage::PeerConnected(peer)) } @@ -388,7 +363,6 @@ pub(crate) fn start_starfish_rbc_service( pending_fetches: AHashMap::new(), staged_notifications: AHashSet::new(), retained_initials: BTreeMap::new(), - retained_transaction_payloads: BTreeMap::new(), retained_phases: BTreeSet::new(), }; let task = tokio::spawn(run_service(state, message_rx, header_retry_interval)); @@ -460,10 +434,6 @@ struct RbcServiceState { /// Recipient-specialized local proposals retained for replay after a /// connection is replaced. Version one keeps these for the run. retained_initials: BTreeMap<(BlockReference, AuthorityIndex), RbcHeaderProposal>, - /// Header-free transaction payloads retained for replay after INIT. The - /// same content-addressed payload is specialized only by its recipient - /// routing, not by its bytes. - retained_transaction_payloads: BTreeMap, /// Authorized local phase intents. Tags are rematerialized for the peer /// on replay rather than retaining or cloning a tagged wire message. retained_phases: BTreeSet<(BlockReference, RbcPhase)>, @@ -495,9 +465,6 @@ impl RbcServiceState { RbcServiceMessage::HeaderResponse { peer, header } => { self.accept_header_response(peer, header); } - RbcServiceMessage::TransactionPayload { peer, payload } => { - self.accept_transaction_payload(peer, payload); - } RbcServiceMessage::PeerConnected(peer) => self.peer_connected(peer), RbcServiceMessage::PeerDisconnected(peer) => self.peer_disconnected(peer), RbcServiceMessage::AdvanceLocalRound { round, reply } => { @@ -528,26 +495,15 @@ impl RbcServiceState { header.transactions_commitment, )?; let canonical = local.header().clone(); - let proposals = self.make_initial_proposals(&local); + let transaction_data = transaction_data.map(Arc::new); + let proposals = self.make_initial_proposals(&local, transaction_data); let (pinned, effects) = local.into_parts(); - let transaction_payload = - transaction_data.map(|data| RbcTransactionPayload::new(canonical.reference(), data)); - if let Some(payload) = transaction_payload.as_ref() { - self.retained_transaction_payloads - .insert(canonical.reference(), payload.clone()); - } self.notify_header_staged(pinned); for (recipient, proposal) in proposals { self.retained_initials .insert((canonical.reference(), recipient), proposal.clone()); self.send_network(recipient, NetworkMessage::RbcInitial(proposal)); - if let Some(payload) = transaction_payload.as_ref() { - self.send_network( - recipient, - NetworkMessage::RbcTransactionPayload(payload.clone()), - ); - } } self.process_effects(effects); Ok(canonical) @@ -556,6 +512,7 @@ impl RbcServiceState { fn make_initial_proposals( &self, local: &RbcLocalInitial, + transaction_data: Option>, ) -> Vec<(AuthorityIndex, RbcHeaderProposal)> { let header = local.header().clone(); match &self.initial_authenticator { @@ -565,7 +522,7 @@ impl RbcServiceState { .make_local_initial_signature_digest(local) .expect("local RBC handle must remain selected"); let proof = RbcInitialProof::Ed25519(signer.sign_digest(&digest)); - self.public_initial_proposals(header, proof) + self.public_initial_proposals(header, proof, transaction_data) } RbcInitialAuthenticator::MlDsa44(signer) => { let digest = self @@ -574,7 +531,7 @@ impl RbcServiceState { .expect("local RBC handle must remain selected"); let proof = RbcInitialProof::MlDsa44(signer.sign_digest(&BlockDigest::from(digest))); - self.public_initial_proposals(header, proof) + self.public_initial_proposals(header, proof, transaction_data) } RbcInitialAuthenticator::MlDsa65(signer) => { let digest = self @@ -583,7 +540,7 @@ impl RbcServiceState { .expect("local RBC handle must remain selected"); let proof = RbcInitialProof::MlDsa65(signer.sign_digest(&BlockDigest::from(digest))); - self.public_initial_proposals(header, proof) + self.public_initial_proposals(header, proof, transaction_data) } RbcInitialAuthenticator::Mac => self .committee @@ -596,7 +553,11 @@ impl RbcServiceState { .expect("local RBC handle must remain selected"); ( recipient, - RbcHeaderProposal::new(header.clone(), RbcInitialProof::Mac(tag)), + RbcHeaderProposal::with_transaction_data( + header.clone(), + RbcInitialProof::Mac(tag), + transaction_data.clone(), + ), ) }) .collect(), @@ -607,6 +568,7 @@ impl RbcServiceState { &self, header: RbcCanonicalHeader, proof: RbcInitialProof, + transaction_data: Option>, ) -> Vec<(AuthorityIndex, RbcHeaderProposal)> { self.committee .authorities() @@ -614,25 +576,31 @@ impl RbcServiceState { .map(|recipient| { ( recipient, - RbcHeaderProposal::new(header.clone(), proof.clone()), + RbcHeaderProposal::with_transaction_data( + header.clone(), + proof.clone(), + transaction_data.clone(), + ), ) }) .collect() } fn accept_direct_initial(&mut self, peer: AuthorityIndex, proposal: RbcHeaderProposal) { - let (header, proof) = proposal.into_parts(); + let (header, proof, transaction_data) = proposal.into_parts(); let block_ref = header.reference(); match self .kernel .accept_direct_initial_header(peer, header, &proof) { Ok(RbcInitialHeaderOutcome::Authenticated { effects }) => { - self.finish_header_staging(block_ref, Some(peer)); + let pinned = self.finish_header_staging(block_ref, Some(peer)); + self.notify_transaction_payload(peer, pinned, transaction_data); self.process_effects(effects); } Ok(RbcInitialHeaderOutcome::StagedUnauthenticated { effects, error }) => { - self.finish_header_staging(block_ref, Some(peer)); + let pinned = self.finish_header_staging(block_ref, Some(peer)); + self.notify_transaction_payload(peer, pinned, transaction_data); self.process_effects(effects); self.reject(Some(peer), error.into()); } @@ -693,43 +661,40 @@ impl RbcServiceState { } } - fn accept_transaction_payload(&mut self, peer: AuthorityIndex, payload: RbcTransactionPayload) { - let block_ref = payload.block_reference(); - if !self.committee.known_authority(peer) { - self.reject(Some(peer), RbcError::UnknownAuthority(peer).into()); - return; - } - if peer != block_ref.authority { - self.reject( - Some(peer), - RbcServiceError::TransactionPayloadNotFromAuthor { block_ref, peer }, - ); - return; - } + fn finish_header_staging( + &mut self, + block_ref: BlockReference, + peer: Option, + ) -> Option { match self.kernel.pinned_header(block_ref) { Ok(Some(header)) => { - let _ = self.events.send(RbcServiceEvent::TransactionPayloadStaged { - peer, - header, - payload, - }); + self.pending_fetches.remove(&block_ref); + self.notify_header_staged(header.clone()); + Some(header) + } + Ok(None) => { + self.reject(peer, RbcError::HeaderUnavailable(block_ref).into()); + None + } + Err(error) => { + self.reject(peer, error.into()); + None } - Ok(None) => self.reject( - Some(peer), - RbcServiceError::UnexpectedTransactionPayload(block_ref), - ), - Err(error) => self.reject(Some(peer), error.into()), } } - fn finish_header_staging(&mut self, block_ref: BlockReference, peer: Option) { - match self.kernel.pinned_header(block_ref) { - Ok(Some(header)) => { - self.pending_fetches.remove(&block_ref); - self.notify_header_staged(header); - } - Ok(None) => self.reject(peer, RbcError::HeaderUnavailable(block_ref).into()), - Err(error) => self.reject(peer, error.into()), + fn notify_transaction_payload( + &self, + peer: AuthorityIndex, + header: Option, + transaction_data: Option>, + ) { + if let (Some(header), Some(transaction_data)) = (header, transaction_data) { + let _ = self.events.send(RbcServiceEvent::TransactionPayloadStaged { + peer, + header, + transaction_data, + }); } } @@ -851,15 +816,6 @@ impl RbcServiceState { self.send_network(peer, NetworkMessage::RbcInitial(proposal)); } - let payloads: Vec<_> = self - .retained_transaction_payloads - .values() - .cloned() - .collect(); - for payload in payloads { - self.send_network(peer, NetworkMessage::RbcTransactionPayload(payload)); - } - let phases: Vec<_> = self.retained_phases.iter().copied().collect(); for (block_ref, phase) in phases { match self.kernel.make_phase_message(phase, block_ref, peer) { @@ -1061,7 +1017,7 @@ mod tests { } #[tokio::test] - async fn local_payload_is_header_free_and_replayed_after_initial() { + async fn local_payload_is_embedded_in_initial_and_replayed_before_phase() { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::MacVector); let transaction_data = TransactionData::new(Vec::new()); let canonical = handle @@ -1079,20 +1035,15 @@ mod tests { RbcServiceEvent::Network { recipient, message: NetworkMessage::RbcInitial(ref proposal), - } if recipient == expected_recipient && proposal.header() == &canonical - )); - assert!(matches!( - next_event(&mut events).await, - RbcServiceEvent::Network { - recipient, - message: NetworkMessage::RbcTransactionPayload(ref payload), } if recipient == expected_recipient - && payload.block_reference() == canonical.reference() + && proposal.header() == &canonical + && proposal.transaction_data().is_some() )); } // Drain the three initial ECHO messages, then reconnect one peer. The - // replay FIFO must put INIT and payload before the rematerialized ECHO. + // replay FIFO must put payload-bearing INIT before the rematerialized + // ECHO. for _ in 0..3 { assert!(matches!( next_event(&mut events).await, @@ -1107,15 +1058,8 @@ mod tests { next_event(&mut events).await, RbcServiceEvent::Network { recipient: 2, - message: NetworkMessage::RbcInitial(_), - } - )); - assert!(matches!( - next_event(&mut events).await, - RbcServiceEvent::Network { - recipient: 2, - message: NetworkMessage::RbcTransactionPayload(ref payload), - } if payload.block_reference() == canonical.reference() + message: NetworkMessage::RbcInitial(ref proposal), + } if proposal.transaction_data().is_some() )); assert!(matches!( next_event(&mut events).await, @@ -1130,34 +1074,58 @@ mod tests { } #[tokio::test] - async fn transaction_payload_requires_its_pinned_header_and_direct_author() { - let (handle, mut events, task) = start_service(1, BlockAuthenticationScheme::MacVector); - let block_ref = BlockReference::new_test(0, 1); - let payload = RbcTransactionPayload::new(block_ref, TransactionData::new(Vec::new())); - - handle.transaction_payload(0, payload.clone()).unwrap(); + async fn direct_initial_stages_header_and_payload_before_echo() { + let (author, mut author_events, author_task) = + start_service(0, BlockAuthenticationScheme::MacVector); + author + .start_local_header_with_payload( + local_header(1, 4), + Some(TransactionData::new(Vec::new())), + ) + .await + .unwrap(); assert!(matches!( - next_event(&mut events).await, - RbcServiceEvent::Rejected { - peer: Some(0), - error: RbcServiceError::UnexpectedTransactionPayload(reference), - } if reference == block_ref + next_event(&mut author_events).await, + RbcServiceEvent::HeaderStaged(_) )); + let proposal = loop { + if let RbcServiceEvent::Network { + recipient: 1, + message: NetworkMessage::RbcInitial(proposal), + } = next_event(&mut author_events).await + { + break proposal; + } + }; - handle.transaction_payload(2, payload).unwrap(); + let (receiver, mut receiver_events, receiver_task) = + start_service(1, BlockAuthenticationScheme::MacVector); + receiver.direct_initial(0, proposal).unwrap(); + let staged = match next_event(&mut receiver_events).await { + RbcServiceEvent::HeaderStaged(header) => header, + other => panic!("expected staged header, got {other:?}"), + }; assert!(matches!( - next_event(&mut events).await, - RbcServiceEvent::Rejected { - peer: Some(2), - error: RbcServiceError::TransactionPayloadNotFromAuthor { - block_ref: reference, - peer: 2, - }, - } if reference == block_ref + next_event(&mut receiver_events).await, + RbcServiceEvent::TransactionPayloadStaged { + peer: 0, + ref header, + ref transaction_data, + } if header.reference() == staged.reference() + && transaction_data.number_transactions() == 0 + )); + assert!(matches!( + next_event(&mut receiver_events).await, + RbcServiceEvent::Network { + message: NetworkMessage::RbcPhase(ref message), + .. + } if message.phase() == RbcPhase::Echo )); - drop(handle); - task.await.unwrap(); + drop(author); + drop(receiver); + author_task.await.unwrap(); + receiver_task.await.unwrap(); } #[tokio::test] diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 264cb171..9620da6b 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -700,6 +700,15 @@ pub struct TransactionData { pub(crate) serialized: Option, } +impl fmt::Debug for TransactionData { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("TransactionData") + .field("transaction_count", &self.transactions.len()) + .finish() + } +} + impl TransactionData { pub fn new(transactions: Vec) -> Self { Self { diff --git a/docs/starfish-rbc-protocol.md b/docs/starfish-rbc-protocol.md index bef38115..49809d2e 100644 --- a/docs/starfish-rbc-protocol.md +++ b/docs/starfish-rbc-protocol.md @@ -259,6 +259,7 @@ HeaderProposal { slot, canonical_header, initial_authentication, + transaction_data?, } RbcPhaseMessage { @@ -279,10 +280,6 @@ HeaderResponse { canonical_header, } -TransactionPayload { - block_ref, - transaction_data, -} ``` The protocol instance, committee ID, and authentication mode are fixed service context and need not @@ -303,13 +300,13 @@ content digest and validating the header's structure. needs nor confers a valid local initial proof. It supplies the content bytes needed for a transition that is authorized by the receiver's own directly authenticated RBC evidence. -The RBC header-retrieval path never asserts transaction-payload availability. Direct transaction -data uses a header-free payload keyed by the already authenticated `BlockReference`; the receiver -checks it against the transaction commitment in the pinned header, derives its local shard, and -attaches or buffers it through the existing Starfish transaction-data path. Shard relay and -reconstruction remain unchanged. Explicit missing-parent synchronization may still return a -header as a recovery fallback, but proactive full-block/header batches are disabled for -Starfish-RBC. +The RBC header-retrieval path never asserts transaction-payload availability. Direct INIT may +co-carry transaction data in the same envelope as the one canonical header; the receiver checks it +against the transaction commitment in the pinned header, derives its local shard, and attaches or +buffers it through the existing Starfish transaction-data path. The payload is not part of the RBC +authentication sidecar or header identity. Shard relay and reconstruction remain unchanged. +Explicit missing-parent synchronization may still return a header as a recovery fallback, but +proactive full-block/header batches are disabled for Starfish-RBC. ## 5. Local state @@ -725,11 +722,11 @@ overhead number: saved INIT bytes can hide part of the phase-message cost. The b INIT/header bytes and ECHO/READY bytes separately; a one-tag lower-bound projection may be added but must be labeled as such. -Starfish-RBC sends the canonical header through INIT (or explicit header recovery) and transports -direct transaction data separately as `RbcTransactionPayload { block_ref, transaction_data }`. -This removes the steady-state duplicate header that the first integrated prototype carried inside -an ordinary full-block batch. The per-message framed-byte counters report the payload and -ECHO/READY costs separately. +Starfish-RBC sends the canonical header once through INIT (or explicit header recovery), with +optional direct transaction data in the same INIT envelope. This removes the steady-state duplicate +header that the first integrated prototype carried inside an ordinary full-block batch without +opening a second scheduling gap before the payload. The per-message framed-byte counters report +the combined INIT bytes and ECHO/READY costs separately. Metrics should separate: @@ -785,7 +782,7 @@ Each milestone is committed separately. authentication, the network RBC service and durable multi-holder fetch retry, dirty/clean lifecycle, clean-only acknowledgments, clean-only consensus/linearization, per-peer ordered outbound isolation, - header-free transaction payloads, and fresh per-run protocol-instance distribution. + single-header INIT transaction transport, and fresh per-run protocol-instance distribution. 5. **End-to-end validation (in progress):** all four initial-authentication modes commit in a four-validator network test, and a late-joining MAC validator catches up. Kernel-level poisoned-tag and equivocation tests are complete; composed dangling-parent and Byzantine From 2bb7ae8233b4e328971f63e023f23225e6f2de75 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:05:52 +0200 Subject: [PATCH 28/62] Document Starfish-RBC-DAG protocol design --- README.md | 5 + docs/starfish-rbc-dag-protocol.md | 790 ++++++++++++++++++++++++++++++ 2 files changed, 795 insertions(+) create mode 100644 docs/starfish-rbc-dag-protocol.md diff --git a/README.md b/README.md index 85d19d07..7ad5fe03 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,11 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). +**Starfish-RBC-DAG** is a design-only follow-up that pipelines all-carrier RBC through an optimistic +carrier DAG while keeping certified Starfish consensus and ordering in a separate logical +projection. Its provisional CLI name is `starfish-rbc-dag`, but that selector is not implemented +yet. The full design and proof obligations are documented in the +[protocol design](docs/starfish-rbc-dag-protocol.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md new file mode 100644 index 00000000..f9af20a9 --- /dev/null +++ b/docs/starfish-rbc-dag-protocol.md @@ -0,0 +1,790 @@ +# Starfish-RBC-DAG protocol design + +Status: design milestone; no implementation or safety/liveness claim yet + +The provisional CLI name for this protocol is `starfish-rbc-dag`. It is a new protocol, not a +transport option or a version-two alias for `starfish-rbc`. + +The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative +baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through +RBC-delivered dependency-closed headers, and sends one initial MAC tag to each recipient. This +document instead specifies an optimistic carrier DAG that embeds the reliable-broadcast transcript, +uses a complete committee-sized MAC vector on every MAC-authenticated carrier, and separates fast +carrier pacing from certified consensus ordering. + +The two designs may share canonicalization, cryptography, storage, and benchmark code, but they are +not wire-compatible and do not have the same proof obligations. Nothing in this document changes +the behavior of `starfish-rbc` or any existing protocol. + +## 1. Objective and non-claims + +The objective is to recover the pipelining of an uncertified Starfish DAG without allowing +optimistically received Byzantine blocks to affect safety: + +- a fast physical carrier DAG advances on a quorum of locally authenticated carriers; +- every application carrier is also the value of a Bracha reliable-broadcast instance; +- later carriers batch ECHO and READY statements for earlier carriers; +- only RBC-delivered, data-available information enters the logical consensus projection; and +- committed consensus frontiers eventually order every honest on-prefix application carrier. + +The initial research mode sends the complete ordered MAC vector with every carrier. The vector is +an authentication sidecar and is not part of the carrier digest. Ed25519, ML-DSA-44, and ML-DSA-65 +remain selectable outer-authentication baselines; changing that selector does not change the +embedded RBC or consensus rules. + +This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish +commit rules already exist, but their composition through two clocks, two logical projections, and +frontier-based payload ordering still requires an executable model, adversarial tests, and a proof. +Until those obligations are discharged, `starfish-rbc-dag` must be described as an experimental +prototype rather than a proven signature-free Starfish variant. + +Transaction bytes remain outside header RBC. The existing Reed-Solomon dissemination, +acknowledgment, reconstruction, and transaction-commitment checks remain responsible for data +availability. + +## 2. Model and notation + +Version one assumes: + +- one static, ordered, stake-weighted committee for a run; +- Byzantine stake strictly below one third of total stake; +- pairwise symmetric keys for every ordered validator pair; +- reliable authenticated point-to-point communication after GST; +- a fresh nonzero protocol-instance identifier shared through genesis configuration; +- no committee reconfiguration; and +- no state retirement until a safe recovery watermark is proved. + +For total committee stake `W`, use the repository's integer thresholds: + +```text +Q = floor(2W / 3) + 1 +V = floor(W / 3) + 1 +``` + +For equal stake and `n = 3f + 1`, these are `Q = 2f + 1` and `V = f + 1`. + +Two independent round numbers are used: + +- `carrier_round` belongs to the fast physical DAG and advances from authenticated admission; +- `consensus_round` belongs to the certified logical Starfish projection. + +There is no fixed mapping between them. Carrier rounds may run ahead while a consensus vertex is +waiting for RBC delivery, data availability, a leader decision, or certified strong parents. + +## 3. One physical DAG, two logical projections + +The only network DAG objects are carriers. A carrier contains application-header data, a bounded +batch of RBC control statements, and optionally one consensus vertex. + +The same stored objects have two disjoint interpretations: + +1. **Optimistic carrier projection.** Authenticated carriers and their weak parent references pace + carrier creation and transport RBC statements. This projection is allowed to differ temporarily + between honest validators. +2. **Certified consensus projection.** Eligible consensus vertices, immutable strong parents, and + certified delivery frontiers drive Starfish voting, certification, skip/commit decisions, and + linearization. Honest validators eventually agree on this projection. + +Weak carrier edges never become strong/order edges, even if their targets later deliver. A node +must not construct its own filtered consensus parent set from an optimistic carrier after the fact: +that would make the authenticated content have different consensus meaning at different nodes. + +This separation prevents a Byzantine carrier from poisoning an honest carrier. A quorum-sized weak +parent set can contain up to `f` selectively disseminated or invented Byzantine references. Waiting +for all such references to become RBC-delivered would make the honest child permanently unusable. +Weak edges are therefore permanently nonblocking and nonordering. Only the explicitly encoded +strong parents and certified frontier constrain consensus. + +## 4. Canonical objects + +The milestone-two codec should implement the following logical types. Field widths, enum codes, +maximum lengths, and golden bytes are frozen by that milestone, before runtime integration. + +```rust +struct CarrierHeaderV1 { + author: AuthorityIndex, + carrier_round: RoundNumber, + + // Physical pacing only. `own_prev` is not repeated in `weak_parents`. + own_prev: BlockReference, + weak_parents: Vec, + + transactions_commitment: TransactionsCommitment, + data_acknowledgments: Vec, + phase_batch: Vec, + consensus_vertex: Option, + creation_time_ns: TimestampNs, +} + +enum RbcPhaseStatementV1 { + Echo { target: BlockReference }, + Ready { target: BlockReference }, +} + +struct ConsensusVertexV1 { + consensus_round: RoundNumber, + strong_parents: Vec, + delivery_frontier: Vec>, + // None only for the fixed genesis consensus round. + leader_choice: Option, +} + +struct ConsensusVertexReference { + carrier: BlockReference, + consensus_round: RoundNumber, +} + +enum LeaderChoiceV1 { + Vote { leader: ConsensusVertexReference }, + NoVote { leader_author: AuthorityIndex, leader_round: RoundNumber }, +} +``` + +The author of an embedded phase statement or consensus vertex is the author of its enclosing +carrier. An outer authenticator therefore authenticates the whole batch without a separate tag or +signature per statement. + +For non-genesis carrier round `r`: + +- `own_prev` has the same author and carrier round `r - 1`; +- every weak parent has carrier round `r - 1` and a distinct non-local author; +- the stake of `{ own_prev } union weak_parents` reaches `Q`; +- phase targets have carrier rounds strictly below `r`; and +- every vector is canonically ordered and duplicate-free. + +Weak references are syntax and pacing declarations, not availability assertions. Their target +headers need not be present to authenticate, admit, process, or RBC-deliver the enclosing carrier. + +Acknowledgments retain Starfish's logical order and compression on the wire. The content digest +commits to the expanded logical vector, while the stored compression must be canonical. An honest +author creates an acknowledgment only after the exact target is locally RBC-delivered and its +transaction data reconstructs to the committed root. The acknowledgment becomes usable as +data-availability evidence only after its enclosing carrier is also locally RBC-delivered; an +optimistically admitted Byzantine carrier cannot create inconsistent availability facts at +different validators. + +`delivery_frontier` has exactly one indexed entry per committee authority. `None` denotes that +authority's fixed genesis/empty prefix. A `Some(reference)` entry must name the same authority as +its vector index. The frontier rules in Section 10 are stateful eligibility rules; they are not +part of context-free carrier decoding. + +### 4.1 Carrier identity + +The carrier reference remains content-only: + +```text +BlockReference = (author, carrier_round, BLAKE3(canonical_carrier_content)) +``` + +Canonical carrier content includes every field of `CarrierHeaderV1`, including the ordered RBC +phase batch and optional consensus vertex. It excludes: + +- the MAC vector or public signature; +- protocol instance and committee ID; +- transaction bytes and erasure-coded shards; +- receipt peer, arrival time, and local admission state; and +- recovery or transport metadata. + +The byte grammar uses fixed field markers, fixed-width integers, and explicit vector lengths. It +does not add a `starfish:block-ref:v2` string to the block identity. A format-version field and the +unambiguous grammar distinguish this carrier layout; changing the layout requires a new version and +new golden vectors. + +Consensus vertices are referenced by their exact enclosing `BlockReference` plus their declared +`consensus_round`. Because there is at most one consensus vertex per carrier, that pair identifies +the immutable embedded value without a second mutable lookup key. + +## 5. Authentication sidecar + +Authentication is outside the carrier reference: + +```rust +enum CarrierAuthenticationV1 { + Ed25519(Ed25519Signature), + MlDsa44(MlDsa44Signature), + MlDsa65(MlDsa65Signature), + MacVector(FlatMacVector), +} + +struct FlatMacVector { + // Exactly committee.len() consecutive 32-byte tags in authority-index order. + tags: Vec, +} +``` + +In MAC mode, author `A` computes entry `q` for recipient `Q_q` over a fixed-width statement that +binds at least: + +```text +STARFISH_RBC_DAG_V1 +carrier-authentication kind and scheme +protocol_instance +committee_id +author A +recipient Q_q +carrier_round +canonical carrier content digest +``` + +The full vector accompanies every normally disseminated MAC carrier in version one, including +relayed carriers. A receiver verifies only the entry at its own committee index. It neither verifies +nor vouches for the remaining entries. + +A carrier received directly from its author and the same carrier received through a relay are both +authentication-eligible when the local entry verifies. This is receiver-specific transferable +authentication: it survives a relay for its intended recipient, but it is not a publicly verifiable +signature and provides no non-repudiation. + +The vector is deliberately not an RBC value and has no consistency invariant. A Byzantine author +may attach different vectors to the same content reference, including a vector with a valid tag for +one recipient and garbage for another. Correctness therefore depends only on the local entry and on +the embedded Bracha protocol, never on agreement about the vector bytes. + +Each node persists one exact vector variant with its carrier for restart and relay. The preference +order is locally generated, directly author-received, then first relayed variant with a valid local +entry. Version one does not merge unverified entries from different vectors. Header recovery after +authenticated quorum phase evidence may return canonical content without a vector; that recovery +can unblock RBC delivery but does not create optimistic carrier admission. + +Public-signature modes use the same context-bound carrier statement without a recipient field and +the same embedded RBC/consensus logic. They exist for controlled performance comparison, not as +separate consensus protocols. + +## 6. Local lifecycle and authority matrix + +The implementation must represent these predicates separately: + +```text +Candidate + canonical carrier content and reference are valid + +Authenticated + local author, valid public signature, or valid local MAC-vector entry + +CarrierAdmitted + Candidate && Authenticated && accepted by the carrier admission window + +Delivered + local Bracha instance reached Q READY and pinned matching canonical content + +PrefixClosed + Delivered && DataAvailable && exact own_prev prefix is closed + +VertexProjected (orthogonal to the carrier lifecycle) + this carrier's optional consensus vertex is eligible in the certified projection + +Included + a committed Starfish anchor frontier names this carrier in its deterministic delta + +Ordered + the complete included delta is available and the carrier has been deterministically output +``` + +`Candidate` alone permits bounded staging and digest-based recovery. `CarrierAdmitted` permits +immediate phase-batch processing and fast-pacemaker counting. `Delivered` permits phase replay even +at a node whose author MAC entry was poisoned. `PrefixClosed` permits frontier inclusion. +`VertexProjected` alone permits the optional vertex to supply Starfish +vote/certifier/leader evidence. It is not a later state of every carrier: a carrier with no eligible +optional vertex may still become prefix-closed, included by another anchor's frontier, and ordered. + +The existing generic `dirty` bit is not a synonym for `CarrierAdmitted`: current RBC code may stage +content after invalid initial authentication or during recovery. Reusing that bit would let an +unauthenticated candidate advance the fast clock. + +| Consumer | Required local authority | +|---|---| +| Header retention/recovery | `Candidate` | +| Process embedded RBC statements | `CarrierAdmitted`, or `Delivered` for replay | +| Fast carrier clock | `CarrierAdmitted` | +| Transaction/shard synchronization | `Candidate` | +| Count a data-availability acknowledgment | `Delivered` enclosing carrier | +| Delivery frontier | `PrefixClosed` target carrier | +| Starfish QC, skip, and anchor commit | `VertexProjected` consensus vertex | +| Application payload output | `Included` delta, with every member delivered/data-available | + +The initial implementation should keep fast-pacemaker counting exactly at `CarrierAdmitted`; using +`Delivered` as an additional stronger pacing input can be added only if the executable model shows +that it cannot change the sequential slot accounting. + +## 7. Fast carrier pacemaker + +The carrier clock is sequential and does not use the current threshold-clock helper's ability to +jump to a far-future round after one message. + +For each carrier round, a validator records at most one admitted reference per author. Byzantine +equivocations may make different honest validators record different references for a Byzantine +author, but each author contributes stake once. A validator advances from carrier round `r` to +`r + 1` only after: + +1. its own carrier at round `r` has been fixed and persisted; and +2. it has admitted distinct-author carrier stake `Q` at round `r`. + +Future carriers are bounded and buffered; they do not skip missing local rounds. The next local +carrier records the selected quorum as `{ own_prev } union weak_parents`. A missing weak-parent body +never blocks the next carrier or any later consensus action. + +This clock replaces the current `starfish-rbc` rule that requires a quorum of RBC-clean previous +round headers before proposal. It does not make admitted carriers consensus votes. Leader/vote/skip +waiting conditions move to the independent consensus projection and cannot block the creation of a +carrier needed to transport ECHO or READY. + +Honest validators emit empty control heartbeats when they have no application transactions. +Without heartbeats, low load can stop the ECHO/READY waves and violate RBC liveness. Every carrier, +including an empty heartbeat, is itself an RBC value and has an authenticator sidecar. + +The first prototype retains the current run's carrier/RBC state and rate-limits carrier creation. +A production design needs a proved runahead and backpressure rule. A hard carrier/consensus skew cap +must not suppress control heartbeats, because those heartbeats may be exactly what allows the +certified frontier to catch up. + +## 8. Embedded all-carrier reliable broadcast + +There is one RBC slot for every physical carrier: + +```text +RbcSlot = (protocol_instance, committee_id, carrier_author, carrier_round) +RbcValue = BlockReference +``` + +Authenticating the carrier is INIT for that value. The local author records its own ECHO when it +atomically fixes and persists the carrier. An honest non-author that first admits a value queues one +ECHO for that slot. + +INIT is not silently counted as the author's ECHO at remote validators. The author's recorded local +ECHO is queued into a later carrier like every other phase action; it counts locally immediately and +remotely only after the enclosing carrier is admitted or delivered. This preserves the standard +quorum accounting without excluding the broadcaster's stake. + +Local phase actions are inserted into the next possible carrier's `phase_batch`. The enclosing +carrier's authentication makes its author the phase sender. For each target slot: + +- an authority emits at most one ECHO; +- an authority emits at most one READY; +- ECHO and READY choices may differ; +- `Q` ECHO stake creates a READY obligation; +- `V` READY stake creates a READY obligation; and +- `Q` READY stake plus pinned matching content locally delivers the exact carrier. + +A READY obligation is not yet a local READY. If the target content is absent, the validator first +recovers it from authenticated ECHO/READY authors, validates the exact reference, and durably pins +it. Only then does it persist the slot-global READY lock, count its local READY, and enqueue the +statement. Consequently every honest READY author is a real content holder. A quorum trigger remains +latched while recovery is pending. + +Local ECHO and READY count toward their thresholds before network dissemination. Evidence is +tracked per candidate, while local send and delivery locks are slot-global. Each remote authority +contributes stake at most once per phase and target slot; exact replay is idempotent and a later +equivocation is ignored before allocating another candidate. + +### 8.1 Processing rule + +After canonical validation and outer authentication, a receiver processes the phase batch in its +canonical encoded order immediately. It must not wait for the enclosing carrier itself to be RBC +delivered, data-available, dependency-closed, or projected. Waiting would create a recursion: a +carrier's controls are required to deliver earlier carriers, while those earlier deliveries may be +required to create the next consensus vertex. + +If the local carrier authenticator is missing or invalid, its phase batch is not processed on +candidate receipt. If that exact outer carrier later becomes locally RBC-delivered, the stored batch +may be replayed under the local delivery capability. Thus poisoned vector entries delay optimistic +admission but cannot permanently suppress controls selected by RBC. + +Phase targets are strictly older carrier rounds, making a single carrier's replay acyclic. Local +arrival order between different authenticated carriers is still observable and can affect which +Byzantine equivocation encounters a slot-global guard first. Recovery must replay the persisted +ingress journal, never reconstruct choices by sorting carriers after a restart. + +### 8.2 Header recovery + +An honest ECHO or READY author must retain the target carrier content. A validator that observes a +threshold before receiving the target requests it from several recorded phase authors. Recovery +content is accepted only when canonical validation recomputes the requested reference. + +Recovery request/response remains an out-of-band data-transfer optimization in the first +prototype. It is not quorum testimony and does not change the on-DAG phase transcript. A `Q` ECHO +set contains honest holders, and a `V` READY set contains at least one honest holder, so retrying +authenticated holders eventually obtains the value after GST. + +### 8.3 Batching and fairness + +Phase batches are bounded. A deterministic fair queue must prevent Byzantine traffic for one slot +from starving honest ECHO/READY actions for other slots. In steady state, one authority can owe one +ECHO and one READY for each of `n` previous-round carriers, so `2n` is the expected arrival rate and +not a safe capacity. The executable model initially uses an unbounded fair queue. A bounded runtime +must reserve strictly more than `2n` statements per carrier, plus an active-slot window, so delayed +work drains instead of remaining at permanent saturation. + +## 9. Certified consensus vertices + +A carrier contains zero or one `ConsensusVertexV1`. The carrier remains valid and pace-eligible if +the optional vertex is malformed relative to local certified state; only the optional vertex is +excluded from the consensus projection. + +A consensus vertex authored by `A` at consensus round `c > 0` is eligible only when: + +1. its enclosing carrier is locally RBC-delivered; +2. its enclosing carrier's transaction data is available and it closes `A`'s carrier prefix as + defined in Section 10; +3. its strong parents name distinct-author eligible consensus vertices at exactly `c - 1` whose + stake reaches `Q`; +4. the strong-parent set includes `A`'s preceding consensus vertex for non-genesis `c`; +5. its delivery frontier is closed and dominates every strong parent's effective frontier; and +6. its leader choice is valid for the deterministic leader role at `c - 1`. + +For a vote, the exact leader must be an eligible strong parent at `c - 1`. No-vote validation is +objective and structural: it names the correct leader slot and no value from that slot appears in +the immutable strong-parent set. Remote validators do not attempt to verify that the author's local +timeout expired. If the strong-parent frontiers contain incomparable components, their join is +undefined and the optional vertex is ineligible; honest construction waits for a compatible quorum +rather than importing the fork into consensus. + +Strong parents and voted leaders decrease strictly in `consensus_round`, which makes the consensus +projection acyclic. Their enclosing `carrier_round` may be numerically higher than the child's +because the clocks are independent and honest carrier authors can be skewed. Strong edges are never +interpreted as physical weak/self edges or application-order dependencies. + +Consensus references and strong parents are immutable authenticated content. Missing or ineligible +strong parents block only this optional vertex. They never block the enclosing carrier, its phase +batch, the fast clock, or later honest RBC progress. + +The consensus pacemaker preserves Starfish's separate advance and creation conditions, evaluated +only over eligible consensus vertices: + +- **A1:** advance from `c - 1` to `c` after eligible distinct-author stake `Q` at `c - 1`; +- **A2:** do not advance until the local consensus vertex at `c - 1` has been fixed; +- **C1:** create at `c` after the eligible leader at `c - 1` is present and the eligible projection + contains either `Q` votes for an exact leader value or a valid explicit direct-skip pattern for + the leader slot at `c - 2`; +- **C2:** create after the consensus leader timeout; or +- **C3:** catch up and create after observing eligible distinct-author stake `Q` already at `c`. + +The strong-parent set chosen under C1 must itself contain the immutable L2 witness: the exact `Q` +voter vertices for a certificate, or the union of explicit negative-choice witnesses required by +the direct-skip evaluator. It must also contain the eligible leader at `c - 1`. Strong-parent sets +therefore contain between `Q` and `n` distinct authors. Merely observing the witness elsewhere in +the local projection is insufficient, because later certifiers must inherit it through the new +vertex's strong history. + +If the eligible leader at `c - 1` is present when the local vertex at `c` is fixed, the vertex must +include that exact leader as a strong parent and record `Vote`. It may record `NoVote` only when it +is created through the timeout/catch-up path without that leader in its immutable strong-parent +set. This timeout/catch-up restriction is an honest-author creation rule; Byzantine authors may +emit structurally valid no-votes arbitrarily. These conditions prevent an adversarially scheduled +quorum that excludes each just-late leader from turning every consensus round into a skip. + +The next local consensus vertex is embedded whenever the carrier scheduler next runs after its +creation condition becomes true. There is no requirement that its carrier round equal `c`, `c + 1`, +or any other fixed offset. Fast carrier production continues while C1/C2/C3 are unsatisfied. + +A Byzantine author may embed conflicting consensus values for the same `(author, consensus_round)` +in different carrier rounds. All structurally eligible conflicts remain visible as equivocation; +there is no local first-arrival or anchor-time pruning rule. An honest author creates at most one +value in its local slot, every strong-parent or evidence set contains at most one value per author, +and stake aggregation counts each author once. Votes and committed leaders name exact references, +so the existing equivocation-aware Starfish safety argument—not an invented canonical +choice—must resolve Byzantine conflicts. + +## 10. Closed delivery prefixes and frontiers + +RBC delivery alone is not a compact availability proof for a Byzantine author's later carrier. A +Byzantine author may deliver round `r` with an `own_prev` that names an unavailable fork at +`r - 1`. Therefore a frontier component is a contiguous exact prefix, not simply the highest +delivered round. + +For authority `A`, begin at its fixed genesis/empty prefix. A carrier `(A, r, R)` extends the local +closed prefix only when: + +- `R` is locally RBC-delivered; +- its transaction data satisfies the existing Starfish availability predicate; +- `r` is exactly one more than the current prefix round; and +- `R.own_prev` equals the exact current prefix tip. + +Later delivered carriers above a gap remain stored but do not advance the prefix. A Byzantine +off-prefix fork may be discarded from application ordering without affecting honest-carrier +liveness. + +The join of strong-parent effective frontiers is computed componentwise. A child frontier dominates +that join only when each entry is the same exact tip or an exact self-chain extension of it; +comparing round numbers alone is insufficient. Including each parent's enclosing carrier prevents a +child from omitting a strong parent from its eventual frontier closure. Honest authors advertise the +newest locally closed tip for every authority, subject to that dominance rule. This monotonic rule +ensures that committed frontiers never regress or switch Byzantine forks. + +The containing carrier cannot name itself in its encoded frontier. For an eligible consensus +vertex, its declared author component must equal its carrier's `own_prev` prefix tip. Once the +enclosing carrier is delivered and data-available, its **effective frontier** replaces that one +component with the enclosing carrier. This makes a committed anchor's own application payload +eligible without waiting for a later anchor while preserving exact prefix continuity. + +The liveness target is deliberately precise: + +> Every honest carrier that RBC-delivers and becomes data-available eventually appears in a +> committed effective-frontier delta. + +No guarantee is made for a malformed or permanently off-prefix Byzantine carrier. Guaranteeing all +RBC-delivered Byzantine forks would require an antichain or sparse exception structure rather than +one compact prefix tip per authority. + +## 11. Starfish certification, commit, and skip + +Starfish's logical leader schedule and commit rules run over eligible consensus vertices only. +Carrier admission, weak parents, phase targets, candidate headers, and merely delivered carriers +cannot act as voters, certifiers, leaders, non-votes, or reachability evidence. + +For a scheduled leader slot at consensus round `c`, every eligible voter publishes one immutable +slot choice. `Vote(L)` is positive evidence only for the exact leader value `L` and explicit +negative evidence for every conflicting value in that leader slot. `NoVote(slot)` is negative +evidence for every value in the slot. Thus a late Byzantine equivocation cannot turn an earlier +omission into a new choice. + +The existing Starfish patterns are then evaluated from these explicit choices: + +- an eligible vertex at `c + 1` explicitly records `Vote(L)` or `NoVote(leader_slot)`; +- `Q` distinct-author votes certify `L`; +- an eligible vertex at `c + 2` whose certified history contains `Q` such votes is a certifier; +- `Q` distinct certifier authors provide the direct-commit condition; and +- a per-candidate quorum of explicit negative choices provides the direct-skip pattern. + +If the leader produces no value, `Q` immutable `NoVote(slot)` choices are a self-contained direct +skip witness. If a Byzantine leader equivocates, `Vote(L)` is negative evidence for every other +candidate, and the current Starfish per-candidate evaluator decides whether the collected explicit +choices form a direct-skip pattern; otherwise the slot remains for indirect decision. + +Indirect commit/skip follows the existing Starfish rule over this same eligible strong-parent +projection. An omission from a phase batch, weak parent list, missing carrier, or locally filtered +view is never a no-vote. `NoVote` is explicit, authenticated, immutable, and slot-locked. +An honest validator persists its leader-choice lock before exposing the carrier that contains it; +it cannot emit `NoVote` and later vote for a late leader in the same logical voting slot. + +Skipping a Byzantine leader role discards only that optional consensus value. It does not discard +the enclosing application carrier. If that carrier later becomes part of a closed prefix, a later +committed frontier orders its payload. + +Every consensus consumer in the current Starfish committer must be audited for the new type +boundary: voter caches, leader support, potential certificates, direct/indirect decisions, +reachability, and the linearizer must reject non-projected carrier facts. Data-availability +acknowledgments are the deliberate exception: they become usable when their enclosing carrier is +RBC-delivered, which breaks a projection/availability circularity while still excluding merely +optimistic evidence. + +## 12. Frontier-delta linearization + +Let `F_k` be the effective frontier carried by committed anchor `A_k`, and let `Closure(F_k)` be the +union of the exact per-author self-chain prefixes named by `F_k`. Maintain: + +```text +C_0 = fixed genesis carriers +C_k = C_(k-1) union Closure(F_k) +Delta = C_k \ C_(k-1) +``` + +Before outputting `Delta`, a validator waits until every exact member is locally RBC-delivered and +data-available. RBC totality and erasure-coded recovery supply missing content for honest committed +frontiers. + +All validators deterministically order the same delta by +`(carrier_round, author, content_digest)`. Because a closed author prefix advances by exactly one +carrier round, this key already preserves mandatory `own_prev` order. + +Weak parents, strong consensus edges, optional-vertex projection time, ECHO/READY target references, +recovery provenance, and MAC-vector variants never constrain application payload ordering. Strong +edges order consensus decisions and dominate frontiers, but a late-projecting optional vertex must +not retroactively add an edge between payloads already output. This fixed ordering also ensures that +a dangling Byzantine weak edge cannot reintroduce the liveness failure that the two-projection +design removes. + +## 13. Expected optimistic schedule + +In an all-honest synchronous interval, batching can realize this conceptual schedule: + +```text +t = 0 carrier k contains a new application header (RBC INIT) +t = delta carrier k+1 contains ECHOs for k +t = 2delta carrier k+2 contains READYs for k +t = 3delta carrier k is RBC-delivered; a later carrier may project new consensus work +``` + +The embedded design does not make Bracha RBC require fewer communication delays than the direct +baseline. Its performance hypothesis is that carrier batching reduces frames, scheduling work, and +duplicated control metadata while the fast carrier clock overlaps certification with dissemination. + +Implementation ordering is latency-critical. On carrier ingress, authenticate, apply its phase +batch, execute newly enabled delivery/prefix/projection transitions, and only then decide what the +next local carrier should contain. Constructing the next carrier first would accidentally add a +full carrier round to every RBC wave. + +The complete vector costs `32n` bytes in every MAC carrier copy. Under all-to-all dissemination this +can erase much of the batching gain. The first benchmark is therefore a whole-protocol result, not +evidence that full-vector all-to-all transport is asymptotically better. Tree or bounded-fanout +vector dissemination is a later optimization and requires redundant routes or direct fallback. + +## 14. Persistence, recovery, and boundedness + +An authoritative implementation must persist proof-critical choices before exposing effects: + +1. journal authenticated inbound provenance and its local ingress sequence; +2. persist local ECHO, READY, explicit no-vote, delivery, carrier-slot, and consensus-slot locks; +3. construct and persist the exact outbound carrier bytes, reference, and authentication sidecar; +4. only then send the carrier; and +5. after restart, replay the journal in recorded order and retransmit the identical carrier. + +Every persisted slot, candidate, lifecycle predicate, journal entry, and outbound-carrier key is +namespaced by both `protocol_instance` and `committee_id`; storage from another run or committee +cannot satisfy a local lock or quorum. + +Hash-sorting recovered carriers is not a valid reconstruction rule. Byzantine equivocation can make +arrival order determine which value a local slot-global guard selects, and a different restart order +could make one honest authority appear to send conflicting phases. + +The initial model and shadow prototype retain all proof-critical carrier, phase, prefix, and +consensus state for the run. Before garbage collection is enabled, the design needs a common +retirement watermark that preserves: + +- pending Bracha totality and header recovery; +- exact self-prefix expansion from the last committed frontier; +- committed-anchor reconstruction for a late validator; and +- deterministic replay of local locks. + +Resource bounds still required before authoritative deployment include a future carrier window, +per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, a bounded payload +runahead policy, and disk-backed recovery. Resource exhaustion is excluded from the initial proof +model but must be measured in the prototype. + +## 15. Safety obligations + +The design is not complete until at least the following claims are proved or falsified by a model: + +1. **Receiver-authentication integrity.** An honest receiver admits a carrier attributed to an + honest author only if that author created the public proof or the receiver's MAC entry. A MAC is + not public non-repudiation, and a Byzantine endpoint knows its own pairwise key. +2. **RBC agreement and integrity.** Slot-global ECHO/READY locks, quorum intersection, and exact + value binding prevent two conflicting carrier values from being delivered by honest validators. +3. **RBC totality.** If one honest validator delivers a value, heartbeats, READY amplification, and + holder recovery cause every honest validator eventually to deliver the same value. +4. **Optimistic isolation.** Carrier admission can change only fast pacing and RBC processing; it + cannot alter a QC, leader decision, skip, commit, acknowledgment certificate, or output order. +5. **Weak-edge non-poisoning.** A missing or equivocating weak parent cannot block delivery, + projection of unrelated honest vertices, or application ordering. +6. **Consensus-slot uniqueness.** Honest validators create/vote once per + `(author, consensus_round)`, and Byzantine conflicts cannot both acquire honest quorum support. +7. **Prefix comparability.** Every accepted frontier component is an exact extension of its strong + ancestors and of every earlier committed component. +8. **Projection safety.** Erasing weak edges and optional consensus metadata that is not + `VertexProjected` leaves a valid execution of the Starfish commit/skip rules over immutable + strong edges; it does not erase otherwise orderable carrier payloads. +9. **Deterministic ordering.** Equal committed anchors imply equal frontier closures, deltas, and + transaction order at all honest validators. +10. **Data availability.** No carrier enters an output delta until its committed transaction root + can be reconstructed and verified. + +## 16. Liveness obligations + +Under partial synchrony and fair processing, the design must establish: + +1. `Q` honest authors continually create authenticated carriers after GST, so the sequential fast + clock advances without Byzantine participation. +2. Empty heartbeat carriers drain every honest ECHO/READY backlog even when application load is + zero. +3. Every honest carrier is RBC-delivered at every honest validator. +4. Existing Starfish data availability eventually closes every honest author's exact carrier + prefix. +5. Honest consensus vertices with quorum strong parents continue to appear despite arbitrary + Byzantine weak parents, malformed optional vertices, and carrier/consensus round skew. +6. The projected Starfish pacemaker eventually commits infinitely many honest anchors. +7. Honest frontier construction is fair: every newly closed honest carrier prefix is eventually + included in a committed frontier. +8. Waiting for a committed delta cannot block forever because every named exact carrier is already + RBC-delivered and data-available by frontier eligibility. + +The guaranteed payload-liveness statement covers every honest on-prefix carrier. Selectively +disseminated, malformed, or off-prefix Byzantine carriers may be ignored. + +## 17. Required executable tests + +Milestone two begins with an isolated deterministic model, not production network wiring. At +minimum it must cover: + +- `n = 4, f = 1` and `n = 7, f = 2` all-honest progress; +- split Byzantine INIT values and receiver-selective poisoned vector entries; +- valid relayed local MAC entries and invalid vector variants; +- ECHO/READY equivocation, replay, reordering, and evidence-before-header recovery; +- zero application load with heartbeat-only RBC completion; +- future carriers that cannot jump the local sequential clock; +- `f` permanently missing weak parents without blocking honest carrier or consensus progress; +- a delivered Byzantine carrier above an unavailable self-chain gap; +- conflicting Byzantine consensus vertices in one logical slot; +- explicit vote/no-vote conflicts and direct plus indirect commit/skip; +- frontier fork, regression, and strong-parent dominance rejection; +- equal committed anchors producing byte-identical output deltas; +- delayed data availability followed by eventual prefix inclusion; +- crash points before and after each persisted lock and outbound-carrier write; and +- shadow replay matching the current direct RBC kernel's delivered references. + +Property tests should mutate every canonical field and verify carrier-reference binding, while +golden tests freeze the version-one encoding and flat vector length. + +## 18. Complexity and benchmark plan + +The first fair benchmark matrix includes: + +- plain Starfish with Ed25519 and each ML-DSA choice; +- the unsafe `starfish-mac` dissemination lower bound; +- implemented direct `starfish-rbc` with the same authentication choices; +- `starfish-rbc-dag` in MAC-vector and signature modes; and +- Sailfish++ as a certified signature-free comparison. + +Hold committee, load, transaction size, topology, latency injection, dissemination fanout, duration, +timeouts, and build constant. Report carrier/INIT, vector, ECHO, READY, recovery, transaction/shard, +and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery +latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained +state. + +Batching can reduce the number of separately scheduled RBC control messages, but it does not remove +their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` +copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout +transport is added. Shadow mode also sends both direct and embedded transcripts and is a correctness +instrument, not a performance result. + +## 19. Contained implementation milestones + +Every milestone is committed separately. + +1. **Protocol specification (this document):** lock the two clocks, lifecycle, full-vector sidecar, + embedded Bracha transitions, certified prefix/frontier, commit/skip boundary, proof obligations, + and experiment plan. No protocol code or CLI selector is added. +2. **Canonical codec and executable model:** add isolated carrier, phase, consensus, frontier, and + sidecar types; golden encodings; a pure in-memory state machine; and deterministic adversarial + simulations. No network or existing consensus path changes. +3. **Persisted shadow carrier path:** build and store carriers alongside the current direct + `starfish-rbc` service, journal ingress and local locks, and compare embedded versus direct RBC + delivery. Direct RBC remains authoritative; shadow results never affect proposals or commits. +4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum + clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses + the current baseline. +5. **Authoritative embedded RBC:** remove direct ECHO/READY authority only after shadow tests show + identical delivery under reordering, loss, equivocation, poisoned tags, and restart. +6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit + leader choice, contiguous delivery frontiers, and strict clean-only committer consumers. +7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct + prefixes and anchors, and add late-node and crash/restart tests. +8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, + signature Starfish variants, and Sailfish++ before attempting tree dissemination. +9. **Tree dissemination:** distribute vector sub-bundles with redundant routing and a direct timeout + fallback; do not change RBC or consensus semantics. + +## 20. Decisions intentionally deferred + +The following values are not safe to guess in the documentation milestone and must be resolved by +the executable model or measured prototype: + +- exact canonical field widths and maximum phase-batch size; +- maximum future-carrier buffer and payload runahead; +- the control-heartbeat rate under low load and backpressure; +- a safe state-retirement, garbage-collection, and late-catch-up watermark; +- whether all supported storage backends are required before authoritative mode; +- quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and +- the tree topology, redundancy, and fallback timers. + +Mixed `starfish-rbc`, `starfish-rbc-dag`, and version-one/version-two deployments must be rejected +by protocol-instance negotiation. The provisional `starfish-rbc-dag` selector is added only after +the codec/model milestone establishes a distinct stable version. From e29647900d69341c128f0c81aadf439b09e448ca Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:05:22 +0200 Subject: [PATCH 29/62] Add Starfish-RBC-DAG codec and executable model --- README.md | 11 +- crates/starfish-core/src/crypto.rs | 12 + crates/starfish-core/src/lib.rs | 1 + .../src/starfish_rbc_dag/journal.rs | 2220 +++++++++++++++ .../starfish-core/src/starfish_rbc_dag/mod.rs | 2516 +++++++++++++++++ .../src/starfish_rbc_dag/model.rs | 2266 +++++++++++++++ .../src/starfish_rbc_dag/projection.rs | 1548 ++++++++++ docs/starfish-rbc-dag-protocol.md | 142 +- 8 files changed, 8674 insertions(+), 42 deletions(-) create mode 100644 crates/starfish-core/src/starfish_rbc_dag/journal.rs create mode 100644 crates/starfish-core/src/starfish_rbc_dag/mod.rs create mode 100644 crates/starfish-core/src/starfish_rbc_dag/model.rs create mode 100644 crates/starfish-core/src/starfish_rbc_dag/projection.rs diff --git a/README.md b/README.md index 7ad5fe03..3408e20c 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,12 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is a design-only follow-up that pipelines all-carrier RBC through an optimistic -carrier DAG while keeping certified Starfish consensus and ordering in a separate logical -projection. Its provisional CLI name is `starfish-rbc-dag`, but that selector is not implemented -yet. The full design and proof obligations are documented in the -[protocol design](docs/starfish-rbc-dag-protocol.md). +**Starfish-RBC-DAG** is a codec-and-model-only follow-up that pipelines all-carrier RBC through an +optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate +logical projection. Its canonical types and deterministic executable models are implemented in +isolation, but there is no network/runtime path and no safety or liveness claim. Its provisional CLI +name is `starfish-rbc-dag`, but that selector is not implemented yet. The design, current boundary, +and proof obligations are documented in the [protocol design](docs/starfish-rbc-dag-protocol.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 8fe6bbf6..c7f3d7b7 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -128,6 +128,11 @@ impl Hasher for Blake3 { } impl TransactionsCommitment { + /// Construct a transaction commitment from its canonical 32-byte form. + pub const fn from_bytes(bytes: [u8; TRANSACTIONS_DIGEST_SIZE]) -> Self { + Self(bytes) + } + pub fn new_from_encoded_transactions( encoded_transactions: &Vec, authority_index: usize, @@ -913,6 +918,13 @@ impl AsRef<[u8]> for SignatureBytes { } } +impl SignatureBytes { + /// Construct an Ed25519 signature from its canonical fixed-width form. + pub const fn from_bytes(bytes: [u8; SIGNATURE_SIZE]) -> Self { + Self(bytes) + } +} + impl AsBytes for TransactionsCommitment { fn as_bytes(&self) -> &[u8] { &self.0 diff --git a/crates/starfish-core/src/lib.rs b/crates/starfish-core/src/lib.rs index bd6068cc..70c7f628 100644 --- a/crates/starfish-core/src/lib.rs +++ b/crates/starfish-core/src/lib.rs @@ -30,6 +30,7 @@ mod rocks_store; mod runtime; pub mod shard_reconstructor; pub mod starfish_rbc; +pub mod starfish_rbc_dag; mod starfish_rbc_service; mod stat; mod state; diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs new file mode 100644 index 00000000..fa7287a6 --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -0,0 +1,2220 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Crash/restart model for proof-critical Starfish-RBC-DAG state. +//! +//! This module is intentionally a deterministic write-ahead-log reducer. It +//! does not perform I/O and it does not duplicate carrier or authentication +//! decoding. Callers validate canonical bytes before journaling them; the +//! reducer pins the exact byte strings and rejects any later alternative. + +use std::{collections::BTreeMap, error::Error, fmt}; + +use crate::types::{AuthorityIndex, BlockReference, RoundNumber}; + +use super::{ + AuthenticatedCarrierV1, CandidateCarrierV1, LeaderChoiceV1, LocallyAuthenticatedCarrierV1, + RbcDagContextV1, RbcPhaseStatementV1, +}; + +/// A Bracha slot is identified independently of the candidate digest. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct RbcSlotKeyV1 { + pub author: AuthorityIndex, + pub carrier_round: RoundNumber, +} + +impl RbcSlotKeyV1 { + pub const fn new(author: AuthorityIndex, carrier_round: RoundNumber) -> Self { + Self { + author, + carrier_round, + } + } + + pub const fn of(reference: BlockReference) -> Self { + Self::new(reference.authority, reference.round) + } +} + +/// Provenance retained for an authenticated ingress record. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IngressProvenanceV1 { + DirectFromAuthor, + Relayed { peer: AuthorityIndex }, +} + +/// One authenticated arrival in its locally observed order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AuthenticatedIngressRecordV1 { + sequence: u64, + reference: BlockReference, + provenance: IngressProvenanceV1, + canonical_carrier_wire: Vec, + authentication_sidecar: Vec, +} + +impl AuthenticatedIngressRecordV1 { + pub fn sequence(&self) -> u64 { + self.sequence + } + + pub fn reference(&self) -> BlockReference { + self.reference + } + + pub fn provenance(&self) -> IngressProvenanceV1 { + self.provenance + } + + pub fn canonical_carrier_wire(&self) -> &[u8] { + &self.canonical_carrier_wire + } + + pub fn authentication_sidecar(&self) -> &[u8] { + &self.authentication_sidecar + } +} + +/// Exact local carrier bytes retained for first send and retransmission. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DurableOutboundCarrierV1 { + reference: BlockReference, + canonical_carrier_wire: Vec, + authentication_sidecar: Vec, + exposed: bool, +} + +impl DurableOutboundCarrierV1 { + pub fn reference(&self) -> BlockReference { + self.reference + } + + pub fn canonical_carrier_wire(&self) -> &[u8] { + &self.canonical_carrier_wire + } + + pub fn authentication_sidecar(&self) -> &[u8] { + &self.authentication_sidecar + } + + pub fn exposed(&self) -> bool { + self.exposed + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum PhaseKindV1 { + Echo, + Ready, +} + +/// Durable result of processing one entry in an enclosing phase batch. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AppliedPhaseOutcomeV1 { + Counted, + IgnoredReplay, + IgnoredEquivocation, +} + +impl PhaseKindV1 { + const fn of(statement: RbcPhaseStatementV1) -> Self { + match statement { + RbcPhaseStatementV1::Echo { .. } => Self::Echo, + RbcPhaseStatementV1::Ready { .. } => Self::Ready, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct SenderPhaseKeyV1 { + slot: RbcSlotKeyV1, + sender: AuthorityIndex, + phase: PhaseKindV1, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct AppliedPhaseEntryV1 { + sender: AuthorityIndex, + statement: RbcPhaseStatementV1, + outcome: AppliedPhaseOutcomeV1, +} + +/// One durable write-ahead event. +/// +/// The context is repeated in every event so that copied records from another +/// protocol instance, committee, or authentication run fail closed on replay. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum JournalEventV1 { + AuthenticatedIngress { + context: RbcDagContextV1, + sequence: u64, + authenticated: AuthenticatedCarrierV1, + provenance: IngressProvenanceV1, + }, + /// Canonical content retained through header recovery. This is content + /// availability for READY/delivery, not authenticated admission for ECHO. + RetainCandidateContent { + context: RbcDagContextV1, + candidate: CandidateCarrierV1, + }, + FixOwnCarrier { + context: RbcDagContextV1, + reference: BlockReference, + }, + LockEcho { + context: RbcDagContextV1, + target: BlockReference, + }, + LockAdmission { + context: RbcDagContextV1, + target: BlockReference, + }, + LockReady { + context: RbcDagContextV1, + target: BlockReference, + }, + LockDelivery { + context: RbcDagContextV1, + target: BlockReference, + }, + LockConsensusSlot { + context: RbcDagContextV1, + consensus_round: RoundNumber, + enclosing_carrier: BlockReference, + }, + LockLeaderChoice { + context: RbcDagContextV1, + consensus_round: RoundNumber, + choice: LeaderChoiceV1, + }, + PersistOutboundContent { + context: RbcDagContextV1, + candidate: CandidateCarrierV1, + }, + PersistOutboundSidecar { + context: RbcDagContextV1, + authenticated: LocallyAuthenticatedCarrierV1, + }, + ExposeOutbound { + context: RbcDagContextV1, + reference: BlockReference, + }, + ApplyPhaseStatement { + context: RbcDagContextV1, + outer: BlockReference, + index: usize, + sender: AuthorityIndex, + statement: RbcPhaseStatementV1, + }, + AdvancePhaseBatchCursor { + context: RbcDagContextV1, + outer: BlockReference, + index: usize, + }, +} + +impl JournalEventV1 { + fn context(&self) -> RbcDagContextV1 { + match self { + Self::AuthenticatedIngress { context, .. } + | Self::RetainCandidateContent { context, .. } + | Self::FixOwnCarrier { context, .. } + | Self::LockEcho { context, .. } + | Self::LockAdmission { context, .. } + | Self::LockReady { context, .. } + | Self::LockDelivery { context, .. } + | Self::LockConsensusSlot { context, .. } + | Self::LockLeaderChoice { context, .. } + | Self::PersistOutboundContent { context, .. } + | Self::PersistOutboundSidecar { context, .. } + | Self::ExposeOutbound { context, .. } + | Self::ApplyPhaseStatement { context, .. } + | Self::AdvancePhaseBatchCursor { context, .. } => *context, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PartialOutboundCarrierV1 { + reference: BlockReference, + candidate: CandidateCarrierV1, + canonical_carrier_wire: Vec, + authentication_sidecar: Option>, + exposed: bool, +} + +impl PartialOutboundCarrierV1 { + fn new(candidate: CandidateCarrierV1, canonical_carrier_wire: Vec) -> Self { + Self { + reference: candidate.reference(), + candidate, + canonical_carrier_wire, + authentication_sidecar: None, + exposed: false, + } + } + + fn complete(&self) -> Option { + Some(DurableOutboundCarrierV1 { + reference: self.reference, + canonical_carrier_wire: self.canonical_carrier_wire.clone(), + authentication_sidecar: self.authentication_sidecar.clone()?, + exposed: self.exposed, + }) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RetainedCarrierV1 { + candidate: CandidateCarrierV1, + canonical_carrier_wire: Vec, +} + +/// State reconstructed exclusively from the durable event sequence. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct JournalSnapshotV1 { + context: RbcDagContextV1, + own_authority: AuthorityIndex, + ingress: Vec, + retained_carriers: BTreeMap, + own_carriers: BTreeMap, + admission_locks: BTreeMap, + echo_locks: BTreeMap, + ready_locks: BTreeMap, + delivery_locks: BTreeMap, + consensus_slots: BTreeMap, + leader_choices: BTreeMap, + outbound: BTreeMap, + applied_phase_entries: BTreeMap<(BlockReference, usize), AppliedPhaseEntryV1>, + sender_phase_locks: BTreeMap, + phase_batch_cursors: BTreeMap, +} + +impl JournalSnapshotV1 { + fn new(context: RbcDagContextV1, own_authority: AuthorityIndex) -> Self { + Self { + context, + own_authority, + ingress: Vec::new(), + retained_carriers: BTreeMap::new(), + own_carriers: BTreeMap::new(), + admission_locks: BTreeMap::new(), + echo_locks: BTreeMap::new(), + ready_locks: BTreeMap::new(), + delivery_locks: BTreeMap::new(), + consensus_slots: BTreeMap::new(), + leader_choices: BTreeMap::new(), + outbound: BTreeMap::new(), + applied_phase_entries: BTreeMap::new(), + sender_phase_locks: BTreeMap::new(), + phase_batch_cursors: BTreeMap::new(), + } + } + + pub fn context(&self) -> RbcDagContextV1 { + self.context + } + + pub fn own_authority(&self) -> AuthorityIndex { + self.own_authority + } + + pub fn authenticated_ingress(&self) -> &[AuthenticatedIngressRecordV1] { + &self.ingress + } + + pub fn next_ingress_sequence(&self) -> u64 { + self.ingress.len() as u64 + } + + pub fn retained_carrier(&self, reference: BlockReference) -> Option<&[u8]> { + self.retained_carriers + .get(&reference) + .map(|retained| retained.canonical_carrier_wire.as_slice()) + } + + pub fn own_carrier(&self, round: RoundNumber) -> Option { + self.own_carriers.get(&round).copied() + } + + pub fn admission_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.admission_locks.get(&slot).copied() + } + + pub fn echo_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.echo_locks.get(&slot).copied() + } + + pub fn ready_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.ready_locks.get(&slot).copied() + } + + pub fn delivery_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.delivery_locks.get(&slot).copied() + } + + pub fn consensus_slot(&self, round: RoundNumber) -> Option { + self.consensus_slots.get(&round).copied() + } + + pub fn leader_choice(&self, round: RoundNumber) -> Option { + self.leader_choices.get(&round).copied() + } + + pub fn phase_batch_cursor(&self, outer: BlockReference) -> usize { + self.phase_batch_cursors.get(&outer).copied().unwrap_or(0) + } + + pub fn phase_statement_applied(&self, outer: BlockReference, index: usize) -> bool { + self.applied_phase_entries.contains_key(&(outer, index)) + } + + pub fn phase_statement_outcome( + &self, + outer: BlockReference, + index: usize, + ) -> Option { + self.applied_phase_entries + .get(&(outer, index)) + .map(|entry| entry.outcome) + } + + pub fn outbound(&self, reference: BlockReference) -> Option { + self.outbound + .get(&reference) + .and_then(PartialOutboundCarrierV1::complete) + } + + /// Byte-identical records that a restart must retransmit. + pub fn retransmissions(&self) -> Vec { + self.outbound + .values() + .filter_map(PartialOutboundCarrierV1::complete) + .filter(DurableOutboundCarrierV1::exposed) + .collect() + } + + fn apply(&mut self, event: &JournalEventV1) -> Result<(), JournalErrorV1> { + if event.context() != self.context { + return Err(JournalErrorV1::ContextMismatch); + } + + match event { + JournalEventV1::AuthenticatedIngress { + sequence, + authenticated, + provenance, + .. + } => self.apply_ingress(*sequence, authenticated, *provenance), + JournalEventV1::RetainCandidateContent { candidate, .. } => { + self.retain_carrier_content(candidate).map(|_| ()) + } + JournalEventV1::FixOwnCarrier { reference, .. } => self.fix_own_carrier(*reference), + JournalEventV1::LockAdmission { target, .. } => self.lock_admission(*target), + JournalEventV1::LockEcho { target, .. } => self.lock_echo(*target), + JournalEventV1::LockReady { target, .. } => self.lock_ready(*target), + JournalEventV1::LockDelivery { target, .. } => { + self.ensure_retained(*target)?; + if self.ready_lock(RbcSlotKeyV1::of(*target)) != Some(*target) { + return Err(JournalErrorV1::DeliveryWithoutMatchingReady(*target)); + } + Self::lock_candidate(&mut self.delivery_locks, *target, LockKindV1::Delivery) + } + JournalEventV1::LockConsensusSlot { + consensus_round, + enclosing_carrier, + .. + } => self.lock_consensus_slot(*consensus_round, *enclosing_carrier), + JournalEventV1::LockLeaderChoice { + consensus_round, + choice, + .. + } => self.lock_leader_choice(*consensus_round, *choice), + JournalEventV1::PersistOutboundContent { candidate, .. } => { + self.persist_outbound_content(candidate) + } + JournalEventV1::PersistOutboundSidecar { authenticated, .. } => { + self.persist_outbound_sidecar(authenticated) + } + JournalEventV1::ExposeOutbound { reference, .. } => self.expose_outbound(*reference), + JournalEventV1::ApplyPhaseStatement { + outer, + index, + sender, + statement, + .. + } => self.apply_phase_statement(*outer, *index, *sender, *statement), + JournalEventV1::AdvancePhaseBatchCursor { outer, index, .. } => { + self.advance_phase_cursor(*outer, *index) + } + } + } + + fn apply_ingress( + &mut self, + sequence: u64, + authenticated: &AuthenticatedCarrierV1, + provenance: IngressProvenanceV1, + ) -> Result<(), JournalErrorV1> { + let expected = self.next_ingress_sequence(); + if sequence != expected { + return Err(JournalErrorV1::IngressSequence { + expected, + actual: sequence, + }); + } + if authenticated.context() != self.context { + return Err(JournalErrorV1::AuthenticatedIngressContextMismatch); + } + if authenticated.receiver() != self.own_authority { + return Err(JournalErrorV1::AuthenticatedIngressReceiverMismatch { + expected: self.own_authority, + actual: authenticated.receiver(), + }); + } + let reference = authenticated.reference(); + let canonical_carrier_wire = self.retain_carrier_content(authenticated.candidate())?; + let authentication_sidecar = authenticated.authentication().canonical_wire_bytes(); + self.ingress.push(AuthenticatedIngressRecordV1 { + sequence, + reference, + provenance, + canonical_carrier_wire, + authentication_sidecar, + }); + Ok(()) + } + + fn fix_own_carrier(&mut self, reference: BlockReference) -> Result<(), JournalErrorV1> { + if reference.authority != self.own_authority { + return Err(JournalErrorV1::OwnCarrierAuthorMismatch { + expected: self.own_authority, + actual: reference.authority, + }); + } + if reference.round == 0 { + return Err(JournalErrorV1::EncodedGenesisCarrier); + } + if !self.outbound.contains_key(&reference) { + return Err(JournalErrorV1::OutboundContentNotPersisted(reference)); + } + Self::lock_exact( + &mut self.own_carriers, + reference.round, + reference, + JournalErrorV1::ConflictingOwnCarrier(reference.round), + ) + } + + fn lock_echo(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { + self.ensure_retained(target)?; + let admitted = self.admission_lock(RbcSlotKeyV1::of(target)) == Some(target); + let fixed_locally = self.own_carrier(target.round) == Some(target); + if !admitted && !fixed_locally { + return Err(JournalErrorV1::EchoWithoutAdmission(target)); + } + Self::lock_candidate(&mut self.echo_locks, target, LockKindV1::Echo) + } + + fn lock_admission(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { + let authenticated_ingress = self + .ingress + .iter() + .any(|ingress| ingress.reference == target); + if !authenticated_ingress { + return Err(JournalErrorV1::AdmissionWithoutAuthenticatedIngress(target)); + } + Self::lock_candidate(&mut self.admission_locks, target, LockKindV1::Admission) + } + + fn lock_ready(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { + self.ensure_retained(target)?; + Self::lock_candidate(&mut self.ready_locks, target, LockKindV1::Ready) + } + + fn ensure_retained(&self, reference: BlockReference) -> Result<(), JournalErrorV1> { + if self.retained_carriers.contains_key(&reference) { + Ok(()) + } else { + Err(JournalErrorV1::CarrierContentNotRetained(reference)) + } + } + + fn retain_carrier_content( + &mut self, + candidate: &CandidateCarrierV1, + ) -> Result, JournalErrorV1> { + if candidate.committee_id() != self.context.committee_id() { + return Err(JournalErrorV1::CandidateCommitteeMismatch); + } + let reference = candidate.reference(); + let canonical_carrier_wire = candidate + .canonical_wire_bytes() + .map_err(|_| JournalErrorV1::CanonicalCarrierEncoding(reference))?; + match self.retained_carriers.get(&reference) { + Some(existing) + if existing.candidate != *candidate + || existing.canonical_carrier_wire != canonical_carrier_wire => + { + Err(JournalErrorV1::ConflictingRetainedContent(reference)) + } + Some(existing) => Ok(existing.canonical_carrier_wire.clone()), + None => { + self.retained_carriers.insert( + reference, + RetainedCarrierV1 { + candidate: candidate.clone(), + canonical_carrier_wire: canonical_carrier_wire.clone(), + }, + ); + Ok(canonical_carrier_wire) + } + } + } + + fn lock_candidate( + locks: &mut BTreeMap, + target: BlockReference, + kind: LockKindV1, + ) -> Result<(), JournalErrorV1> { + let slot = RbcSlotKeyV1::of(target); + Self::lock_exact( + locks, + slot, + target, + JournalErrorV1::ConflictingPhaseLock { kind, slot }, + ) + } + + fn lock_consensus_slot( + &mut self, + consensus_round: RoundNumber, + enclosing_carrier: BlockReference, + ) -> Result<(), JournalErrorV1> { + if consensus_round == 0 { + return Err(JournalErrorV1::EncodedGenesisConsensusVertex); + } + if enclosing_carrier.authority != self.own_authority + || !self + .own_carriers + .values() + .any(|reference| *reference == enclosing_carrier) + { + return Err(JournalErrorV1::ConsensusCarrierNotFixed(enclosing_carrier)); + } + let matching_vertex = self + .outbound + .get(&enclosing_carrier) + .and_then(|outbound| outbound.candidate.header().consensus_vertex()) + .is_some_and(|vertex| vertex.consensus_round() == consensus_round); + if !matching_vertex { + return Err(JournalErrorV1::ConsensusVertexMismatch { + consensus_round, + enclosing_carrier, + }); + } + Self::lock_exact( + &mut self.consensus_slots, + consensus_round, + enclosing_carrier, + JournalErrorV1::ConflictingConsensusSlot(consensus_round), + ) + } + + fn lock_leader_choice( + &mut self, + consensus_round: RoundNumber, + choice: LeaderChoiceV1, + ) -> Result<(), JournalErrorV1> { + let Some(enclosing_carrier) = self.consensus_slot(consensus_round) else { + return Err(JournalErrorV1::LeaderChoiceWithoutConsensusSlot( + consensus_round, + )); + }; + let matching_choice = self + .outbound + .get(&enclosing_carrier) + .and_then(|outbound| outbound.candidate.header().consensus_vertex()) + .is_some_and(|vertex| { + vertex.consensus_round() == consensus_round && vertex.leader_choice() == choice + }); + if !matching_choice { + return Err(JournalErrorV1::LeaderChoiceCandidateMismatch( + consensus_round, + )); + } + Self::lock_exact( + &mut self.leader_choices, + consensus_round, + choice, + JournalErrorV1::ConflictingLeaderChoice(consensus_round), + ) + } + + fn persist_outbound_content( + &mut self, + candidate: &CandidateCarrierV1, + ) -> Result<(), JournalErrorV1> { + let reference = candidate.reference(); + if reference.authority != self.own_authority { + return Err(JournalErrorV1::OwnCarrierAuthorMismatch { + expected: self.own_authority, + actual: reference.authority, + }); + } + if reference.round == 0 { + return Err(JournalErrorV1::EncodedGenesisCarrier); + } + if self + .own_carrier(reference.round) + .is_some_and(|fixed| fixed != reference) + { + return Err(JournalErrorV1::ConflictingOwnCarrier(reference.round)); + } + let canonical_carrier_wire = self.retain_carrier_content(candidate)?; + match self.outbound.get(&reference) { + Some(existing) + if existing.candidate != *candidate + || existing.canonical_carrier_wire != canonical_carrier_wire => + { + Err(JournalErrorV1::ConflictingOutboundContent(reference)) + } + Some(_) => Ok(()), + None => { + self.outbound.insert( + reference, + PartialOutboundCarrierV1::new(candidate.clone(), canonical_carrier_wire), + ); + Ok(()) + } + } + } + + fn persist_outbound_sidecar( + &mut self, + authenticated: &LocallyAuthenticatedCarrierV1, + ) -> Result<(), JournalErrorV1> { + if authenticated.context() != self.context { + return Err(JournalErrorV1::OutboundAuthenticationContextMismatch); + } + let reference = authenticated.reference(); + if reference.authority != self.own_authority { + return Err(JournalErrorV1::OwnCarrierAuthorMismatch { + expected: self.own_authority, + actual: reference.authority, + }); + } + if self.own_carrier(reference.round) != Some(reference) { + return Err(JournalErrorV1::OwnCarrierNotFixed(reference)); + } + let Some(outbound) = self.outbound.get_mut(&reference) else { + return Err(JournalErrorV1::OutboundContentNotPersisted(reference)); + }; + if outbound.candidate != *authenticated.candidate() { + return Err(JournalErrorV1::OutboundAuthenticationCandidateMismatch( + reference, + )); + } + let authentication_sidecar = authenticated.authentication().canonical_wire_bytes(); + match &outbound.authentication_sidecar { + Some(existing) if *existing != authentication_sidecar => { + Err(JournalErrorV1::ConflictingOutboundSidecar(reference)) + } + Some(_) => Ok(()), + None => { + outbound.authentication_sidecar = Some(authentication_sidecar); + Ok(()) + } + } + } + + fn expose_outbound(&mut self, reference: BlockReference) -> Result<(), JournalErrorV1> { + if self.own_carrier(reference.round) != Some(reference) { + return Err(JournalErrorV1::OwnCarrierNotFixed(reference)); + } + let Some(outbound) = self.outbound.get(&reference) else { + return Err(JournalErrorV1::OutboundContentNotPersisted(reference)); + }; + if outbound.authentication_sidecar.is_none() { + return Err(JournalErrorV1::OutboundSidecarNotPersisted(reference)); + } + let candidate = outbound.candidate.clone(); + if self.echo_lock(RbcSlotKeyV1::of(reference)) != Some(reference) { + return Err(JournalErrorV1::OutboundEchoNotLocked(reference)); + } + for statement in candidate.header().phase_batch() { + let target = statement.target(); + let lock = match statement { + RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), + RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), + }; + if lock != Some(target) { + return Err(JournalErrorV1::OutboundPhaseNotLocked(*statement)); + } + } + if let Some(vertex) = candidate.header().consensus_vertex() { + let consensus_round = vertex.consensus_round(); + if self.consensus_slot(consensus_round) != Some(reference) { + return Err(JournalErrorV1::OutboundConsensusSlotNotLocked { + consensus_round, + reference, + }); + } + if self.leader_choice(consensus_round) != Some(vertex.leader_choice()) { + return Err(JournalErrorV1::OutboundLeaderChoiceNotLocked( + consensus_round, + )); + } + } + self.outbound + .get_mut(&reference) + .expect("outbound remains retained") + .exposed = true; + Ok(()) + } + + fn apply_phase_statement( + &mut self, + outer: BlockReference, + index: usize, + sender: AuthorityIndex, + statement: RbcPhaseStatementV1, + ) -> Result<(), JournalErrorV1> { + if sender != outer.authority { + return Err(JournalErrorV1::PhaseSenderMismatch { + outer_author: outer.authority, + sender, + }); + } + let target = statement.target(); + if target.round >= outer.round { + return Err(JournalErrorV1::PhaseTargetNotOlder { outer, target }); + } + let Some(retained_outer) = self.retained_carriers.get(&outer) else { + return Err(JournalErrorV1::OuterCarrierContentNotRetained(outer)); + }; + if retained_outer + .candidate + .header() + .phase_batch() + .get(index) + .copied() + != Some(statement) + { + return Err(JournalErrorV1::PhaseBatchEntryMismatch { outer, index }); + } + let outer_slot = RbcSlotKeyV1::of(outer); + let authorized = self.own_carrier(outer.round) == Some(outer) + || self.admission_lock(outer_slot) == Some(outer) + || self.delivery_lock(outer_slot) == Some(outer); + if !authorized { + return Err(JournalErrorV1::OuterCarrierNotAdmittedOrDelivered(outer)); + } + + let cursor = self.phase_batch_cursor(outer); + if index > cursor { + return Err(JournalErrorV1::PhaseBatchIndexGap { + outer, + expected: cursor, + actual: index, + }); + } + if let Some(existing) = self.applied_phase_entries.get(&(outer, index)) { + return if existing.sender == sender && existing.statement == statement { + Ok(()) + } else { + Err(JournalErrorV1::ConflictingPhaseBatchEntry { outer, index }) + }; + } + if index < cursor { + return Err(JournalErrorV1::MissingAppliedPhaseEntry { outer, index }); + } + + if sender == self.own_authority { + if self.own_carrier(outer.round) != Some(outer) { + return Err(JournalErrorV1::OwnOuterCarrierNotFixed(outer)); + } + let lock = match statement { + RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), + RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), + }; + if lock != Some(target) { + return Err(JournalErrorV1::OwnPhaseWithoutDurableLock(statement)); + } + } + + let sender_key = SenderPhaseKeyV1 { + slot: RbcSlotKeyV1::of(target), + sender, + phase: PhaseKindV1::of(statement), + }; + let outcome = match self.sender_phase_locks.get(&sender_key) { + Some(existing) if *existing != target => AppliedPhaseOutcomeV1::IgnoredEquivocation, + Some(_) => AppliedPhaseOutcomeV1::IgnoredReplay, + None => { + self.sender_phase_locks.insert(sender_key, target); + AppliedPhaseOutcomeV1::Counted + } + }; + self.applied_phase_entries.insert( + (outer, index), + AppliedPhaseEntryV1 { + sender, + statement, + outcome, + }, + ); + Ok(()) + } + + fn advance_phase_cursor( + &mut self, + outer: BlockReference, + index: usize, + ) -> Result<(), JournalErrorV1> { + let cursor = self.phase_batch_cursor(outer); + if index < cursor { + return if self.applied_phase_entries.contains_key(&(outer, index)) { + Ok(()) + } else { + Err(JournalErrorV1::MissingAppliedPhaseEntry { outer, index }) + }; + } + if index > cursor { + return Err(JournalErrorV1::PhaseBatchIndexGap { + outer, + expected: cursor, + actual: index, + }); + } + if !self.applied_phase_entries.contains_key(&(outer, index)) { + return Err(JournalErrorV1::PhaseCursorBeforeApplication { outer, index }); + } + self.phase_batch_cursors.insert(outer, cursor + 1); + Ok(()) + } + + fn lock_exact( + map: &mut BTreeMap, + key: K, + value: V, + conflict: JournalErrorV1, + ) -> Result<(), JournalErrorV1> + where + K: Ord, + V: Eq, + { + match map.get(&key) { + Some(existing) if *existing != value => Err(conflict), + Some(_) => Ok(()), + None => { + map.insert(key, value); + Ok(()) + } + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LockKindV1 { + Admission, + Echo, + Ready, + Delivery, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum JournalErrorV1 { + ContextMismatch, + IngressSequence { + expected: u64, + actual: u64, + }, + AuthenticatedIngressContextMismatch, + AuthenticatedIngressReceiverMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + CandidateCommitteeMismatch, + CanonicalCarrierEncoding(BlockReference), + CarrierContentNotRetained(BlockReference), + ConflictingRetainedContent(BlockReference), + AdmissionWithoutAuthenticatedIngress(BlockReference), + EchoWithoutAdmission(BlockReference), + OwnCarrierAuthorMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + EncodedGenesisCarrier, + ConflictingOwnCarrier(RoundNumber), + ConflictingPhaseLock { + kind: LockKindV1, + slot: RbcSlotKeyV1, + }, + DeliveryWithoutMatchingReady(BlockReference), + EncodedGenesisConsensusVertex, + ConsensusCarrierNotFixed(BlockReference), + ConsensusVertexMismatch { + consensus_round: RoundNumber, + enclosing_carrier: BlockReference, + }, + ConflictingConsensusSlot(RoundNumber), + LeaderChoiceWithoutConsensusSlot(RoundNumber), + LeaderChoiceCandidateMismatch(RoundNumber), + ConflictingLeaderChoice(RoundNumber), + OwnCarrierNotFixed(BlockReference), + ConflictingOutboundContent(BlockReference), + OutboundContentNotPersisted(BlockReference), + ConflictingOutboundSidecar(BlockReference), + OutboundSidecarNotPersisted(BlockReference), + OutboundAuthenticationContextMismatch, + OutboundAuthenticationCandidateMismatch(BlockReference), + OutboundEchoNotLocked(BlockReference), + OutboundPhaseNotLocked(RbcPhaseStatementV1), + OutboundConsensusSlotNotLocked { + consensus_round: RoundNumber, + reference: BlockReference, + }, + OutboundLeaderChoiceNotLocked(RoundNumber), + PhaseSenderMismatch { + outer_author: AuthorityIndex, + sender: AuthorityIndex, + }, + PhaseTargetNotOlder { + outer: BlockReference, + target: BlockReference, + }, + OuterCarrierContentNotRetained(BlockReference), + OuterCarrierNotAdmittedOrDelivered(BlockReference), + OwnOuterCarrierNotFixed(BlockReference), + PhaseBatchEntryMismatch { + outer: BlockReference, + index: usize, + }, + PhaseBatchIndexGap { + outer: BlockReference, + expected: usize, + actual: usize, + }, + ConflictingPhaseBatchEntry { + outer: BlockReference, + index: usize, + }, + MissingAppliedPhaseEntry { + outer: BlockReference, + index: usize, + }, + OwnPhaseWithoutDurableLock(RbcPhaseStatementV1), + PhaseCursorBeforeApplication { + outer: BlockReference, + index: usize, + }, +} + +impl fmt::Display for JournalErrorV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "Starfish-RBC-DAG journal error: {self:?}") + } +} + +impl Error for JournalErrorV1 {} + +/// A deterministic write-ahead journal with a volatile replayed snapshot. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WriteAheadJournalV1 { + context: RbcDagContextV1, + own_authority: AuthorityIndex, + durable_events: Vec, + snapshot: JournalSnapshotV1, +} + +impl WriteAheadJournalV1 { + pub fn new(context: RbcDagContextV1, own_authority: AuthorityIndex) -> Self { + Self { + context, + own_authority, + durable_events: Vec::new(), + snapshot: JournalSnapshotV1::new(context, own_authority), + } + } + + /// Validate against a cloned snapshot, then atomically make the event + /// durable and visible. A real backend maps this boundary to its durable + /// transaction commit. + pub fn append(&mut self, event: JournalEventV1) -> Result<(), JournalErrorV1> { + let mut next = self.snapshot.clone(); + next.apply(&event)?; + self.durable_events.push(event); + self.snapshot = next; + Ok(()) + } + + pub fn record_authenticated_ingress( + &mut self, + authenticated: AuthenticatedCarrierV1, + provenance: IngressProvenanceV1, + ) -> Result { + let sequence = self.snapshot.next_ingress_sequence(); + self.append(JournalEventV1::AuthenticatedIngress { + context: self.context, + sequence, + authenticated, + provenance, + })?; + Ok(sequence) + } + + pub fn durable_events(&self) -> &[JournalEventV1] { + &self.durable_events + } + + pub fn snapshot(&self) -> &JournalSnapshotV1 { + &self.snapshot + } + + /// Rebuild volatile state in exact durable order. + pub fn restart(&self) -> Result { + Self::from_durable_events( + self.context, + self.own_authority, + self.durable_events.clone(), + ) + } + + pub fn from_durable_events( + context: RbcDagContextV1, + own_authority: AuthorityIndex, + durable_events: Vec, + ) -> Result { + let mut snapshot = JournalSnapshotV1::new(context, own_authority); + for event in &durable_events { + snapshot.apply(event)?; + } + Ok(Self { + context, + own_authority, + durable_events, + snapshot, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + committee::Committee, + crypto::{TransactionsCommitment, mac_keyrings_for_test}, + starfish_rbc_dag::{ + CarrierAuthenticationV1, CarrierAuthorizerV1, CarrierHeaderV1Args, + ConsensusVertexReference, ConsensusVertexV1, RbcDagProtocolInstanceId, + carrier_genesis_reference, + }, + types::{BlockAuthenticationScheme, BlockDigest}, + }; + + fn committee() -> std::sync::Arc { + Committee::new_test(vec![1; 4]) + } + + fn context(marker: u8) -> RbcDagContextV1 { + let committee = committee(); + RbcDagContextV1::new( + RbcDagProtocolInstanceId::new([marker; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ) + .unwrap() + } + + fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { + BlockReference { + authority, + round, + digest: BlockDigest::from([marker; 32]), + } + } + + fn journal() -> WriteAheadJournalV1 { + WriteAheadJournalV1::new(context(0xA1), 1) + } + + fn fix_event(journal: &WriteAheadJournalV1, reference: BlockReference) -> JournalEventV1 { + JournalEventV1::FixOwnCarrier { + context: journal.context, + reference, + } + } + + fn candidate( + author: AuthorityIndex, + round: RoundNumber, + marker: u8, + phase_batch: Vec, + consensus_vertex: Option, + ) -> CandidateCarrierV1 { + let committee = committee(); + let previous_round = round - 1; + let parent = |authority| { + if previous_round == 0 { + carrier_genesis_reference(authority) + } else { + reference(authority, previous_round, 0x40 + authority as u8) + } + }; + let mut parent_stake = committee.get_stake(author).unwrap(); + let mut weak_parents = Vec::new(); + for authority in committee + .authorities() + .filter(|authority| *authority != author) + { + if parent_stake >= committee.quorum_threshold() { + break; + } + parent_stake += committee.get_stake(authority).unwrap(); + weak_parents.push(parent(authority)); + } + CandidateCarrierV1::try_new( + CarrierHeaderV1Args { + author, + carrier_round: round, + own_prev: parent(author), + weak_parents, + transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + data_acknowledgments: Vec::new(), + phase_batch, + consensus_vertex, + creation_time_ns: u64::from(marker), + }, + &committee, + ) + .unwrap() + } + + fn consensus_vertex( + enclosing_author: AuthorityIndex, + consensus_round: RoundNumber, + ) -> ConsensusVertexV1 { + let committee = committee(); + let parent_round = consensus_round - 1; + let parent = |authority| { + let carrier = if parent_round == 0 { + carrier_genesis_reference(authority) + } else { + reference(authority, parent_round, 0xD0 + authority as u8) + }; + ConsensusVertexReference::new(carrier, parent_round) + }; + let strong_parents: Vec<_> = committee.authorities().map(parent).collect(); + assert!( + strong_parents + .iter() + .any(|parent| parent.author() == enclosing_author) + ); + let leader = parent(committee.elect_leader(parent_round)); + ConsensusVertexV1::new( + consensus_round, + strong_parents, + vec![None; committee.len()], + LeaderChoiceV1::Vote { leader }, + ) + } + + fn outbound_content_event( + journal: &WriteAheadJournalV1, + candidate: &CandidateCarrierV1, + ) -> JournalEventV1 { + JournalEventV1::PersistOutboundContent { + context: journal.context, + candidate: candidate.clone(), + } + } + + fn persist_and_fix(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { + journal + .append(outbound_content_event(journal, candidate)) + .unwrap(); + journal + .append(fix_event(journal, candidate.reference())) + .unwrap(); + } + + fn retain_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { + journal + .append(JournalEventV1::RetainCandidateContent { + context: journal.context, + candidate: candidate.clone(), + }) + .unwrap(); + } + + fn authentication_with_marker( + candidate: &CandidateCarrierV1, + context_marker: u8, + ) -> CarrierAuthenticationV1 { + let committee = committee(); + let context = context(context_marker); + let keyrings = mac_keyrings_for_test(committee.len()); + let author = candidate.header().author() as usize; + context + .authenticate( + candidate, + &committee, + CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ) + .unwrap() + } + + fn locally_authenticated_with_marker( + candidate: &CandidateCarrierV1, + context_marker: u8, + ) -> LocallyAuthenticatedCarrierV1 { + let committee = committee(); + let context = context(context_marker); + let keyrings = mac_keyrings_for_test(committee.len()); + let author = candidate.header().author() as usize; + context + .authenticate_local( + candidate.clone(), + &committee, + CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ) + .unwrap() + } + + fn locally_authenticated(candidate: &CandidateCarrierV1) -> LocallyAuthenticatedCarrierV1 { + locally_authenticated_with_marker(candidate, 0xA1) + } + + fn authenticated_with_marker( + candidate: &CandidateCarrierV1, + receiver: AuthorityIndex, + context_marker: u8, + ) -> AuthenticatedCarrierV1 { + let committee = committee(); + let context = context(context_marker); + let keyrings = mac_keyrings_for_test(committee.len()); + context + .verify_authentication( + candidate.clone(), + authentication_with_marker(candidate, context_marker), + receiver, + &committee, + &keyrings[receiver as usize], + ) + .unwrap() + } + + fn authenticated( + candidate: &CandidateCarrierV1, + receiver: AuthorityIndex, + ) -> AuthenticatedCarrierV1 { + authenticated_with_marker(candidate, receiver, 0xA1) + } + + fn authenticate_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { + journal + .record_authenticated_ingress( + authenticated(candidate, journal.own_authority), + IngressProvenanceV1::DirectFromAuthor, + ) + .unwrap(); + } + + fn admit_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { + authenticate_candidate(journal, candidate); + journal + .append(JournalEventV1::LockAdmission { + context: journal.context, + target: candidate.reference(), + }) + .unwrap(); + } + + fn prepare_outbound_for_exposure( + journal: &mut WriteAheadJournalV1, + candidate: &CandidateCarrierV1, + ) { + let reference = candidate.reference(); + persist_and_fix(journal, candidate); + journal + .append(JournalEventV1::LockEcho { + context: journal.context, + target: reference, + }) + .unwrap(); + journal + .append(JournalEventV1::PersistOutboundSidecar { + context: journal.context, + authenticated: locally_authenticated(candidate), + }) + .unwrap(); + } + + fn assert_before_after(journal: &WriteAheadJournalV1, event: JournalEventV1, assertion: F) + where + F: Fn(&JournalSnapshotV1) -> bool, + { + let before = journal.restart().unwrap(); + assert!(!assertion(before.snapshot())); + + let mut after = journal.clone(); + after.append(event).unwrap(); + let after = after.restart().unwrap(); + assert!(assertion(after.snapshot())); + } + + #[test] + fn authenticated_ingress_sequence_and_bytes_survive_restart_in_order() { + let mut journal = journal(); + let first_candidate = candidate(0, 1, 0x10, Vec::new(), None); + let second_candidate = candidate(2, 1, 0x20, Vec::new(), None); + let first = first_candidate.reference(); + let second = second_candidate.reference(); + let first_authenticated = authenticated(&first_candidate, 1); + let second_authenticated = authenticated(&second_candidate, 1); + let first_wire = first_candidate.canonical_wire_bytes().unwrap(); + let second_sidecar = second_authenticated.authentication().canonical_wire_bytes(); + assert_eq!( + journal + .record_authenticated_ingress( + first_authenticated, + IngressProvenanceV1::DirectFromAuthor, + ) + .unwrap(), + 0 + ); + assert_eq!( + journal + .record_authenticated_ingress( + second_authenticated, + IngressProvenanceV1::Relayed { peer: 3 }, + ) + .unwrap(), + 1 + ); + + let restarted = journal.restart().unwrap().restart().unwrap(); + let ingress = restarted.snapshot().authenticated_ingress(); + assert_eq!( + ingress + .iter() + .map(|entry| entry.sequence()) + .collect::>(), + [0, 1] + ); + assert_eq!(ingress[0].reference(), first); + assert_eq!(ingress[1].reference(), second); + assert_eq!(ingress[0].canonical_carrier_wire(), first_wire); + assert_eq!(ingress[1].authentication_sidecar(), second_sidecar); + + let mut corrupt = journal.durable_events().to_vec(); + if let JournalEventV1::AuthenticatedIngress { sequence, .. } = &mut corrupt[1] { + *sequence = 2; + } + assert!(matches!( + WriteAheadJournalV1::from_durable_events(context(0xA1), 1, corrupt), + Err(JournalErrorV1::IngressSequence { + expected: 1, + actual: 2 + }) + )); + } + + #[test] + fn crash_boundaries_preserve_each_slot_global_lock() { + let own_candidate = candidate(1, 1, 0x11, Vec::new(), None); + let target_candidate = candidate(2, 1, 0x21, Vec::new(), None); + let own = own_candidate.reference(); + let target = target_candidate.reference(); + let slot = RbcSlotKeyV1::of(target); + let mut content_base = journal(); + content_base + .append(outbound_content_event(&content_base, &own_candidate)) + .unwrap(); + assert_before_after(&content_base, fix_event(&content_base, own), |state| { + state.own_carrier(1) == Some(own) + }); + + let mut base = content_base; + base.append(fix_event(&base, own)).unwrap(); + authenticate_candidate(&mut base, &target_candidate); + let admission = JournalEventV1::LockAdmission { + context: base.context, + target, + }; + assert_before_after(&base, admission.clone(), |state| { + state.admission_lock(slot) == Some(target) + }); + base.append(admission).unwrap(); + + let echo = JournalEventV1::LockEcho { + context: base.context, + target, + }; + assert_before_after(&base, echo, |state| state.echo_lock(slot) == Some(target)); + + let mut ready_base = base.clone(); + let ready = JournalEventV1::LockReady { + context: base.context, + target, + }; + assert_before_after(&ready_base, ready.clone(), |state| { + state.ready_lock(slot) == Some(target) + }); + ready_base.append(ready).unwrap(); + + let delivery = JournalEventV1::LockDelivery { + context: base.context, + target, + }; + assert_before_after(&ready_base, delivery, |state| { + state.delivery_lock(slot) == Some(target) + }); + } + + #[test] + fn crash_boundaries_preserve_consensus_and_leader_choice_locks() { + let mut journal = journal(); + let vertex = consensus_vertex(1, 1); + let expected_choice = vertex.leader_choice(); + let own_candidate = candidate(1, 2, 0x32, Vec::new(), Some(vertex)); + let own = own_candidate.reference(); + persist_and_fix(&mut journal, &own_candidate); + let consensus = JournalEventV1::LockConsensusSlot { + context: journal.context, + consensus_round: 1, + enclosing_carrier: own, + }; + assert_before_after(&journal, consensus.clone(), |state| { + state.consensus_slot(1) == Some(own) + }); + + journal.append(consensus).unwrap(); + let choice = JournalEventV1::LockLeaderChoice { + context: journal.context, + consensus_round: 1, + choice: expected_choice, + }; + assert_before_after(&journal, choice, |state| { + state.leader_choice(1) == Some(expected_choice) + }); + } + + #[test] + fn outbound_content_sidecar_and_exposure_are_separate_crash_boundaries() { + let mut journal = journal(); + let own_candidate = candidate(1, 1, 0x41, Vec::new(), None); + let own = own_candidate.reference(); + let expected_wire = own_candidate.canonical_wire_bytes().unwrap(); + let own_authenticated = locally_authenticated(&own_candidate); + let expected_sidecar = own_authenticated.authentication().canonical_wire_bytes(); + + let content = JournalEventV1::PersistOutboundContent { + context: journal.context, + candidate: own_candidate, + }; + assert_before_after(&journal, content.clone(), |state| { + state.outbound.contains_key(&own) + }); + assert!(journal.snapshot().retransmissions().is_empty()); + journal.append(content).unwrap(); + + let fix = fix_event(&journal, own); + assert_before_after(&journal, fix.clone(), |state| { + state.own_carrier(1) == Some(own) + }); + journal.append(fix).unwrap(); + journal + .append(JournalEventV1::LockEcho { + context: journal.context, + target: own, + }) + .unwrap(); + + let sidecar = JournalEventV1::PersistOutboundSidecar { + context: journal.context, + authenticated: own_authenticated, + }; + assert_before_after(&journal, sidecar.clone(), |state| { + state.outbound(own).is_some() + }); + journal.append(sidecar).unwrap(); + assert!(journal.snapshot().retransmissions().is_empty()); + + let expose = JournalEventV1::ExposeOutbound { + context: journal.context, + reference: own, + }; + assert_before_after(&journal, expose.clone(), |state| { + state + .outbound(own) + .is_some_and(|outbound| outbound.exposed()) + }); + journal.append(expose).unwrap(); + + let before = journal.snapshot().retransmissions(); + let after = journal.restart().unwrap().snapshot().retransmissions(); + assert_eq!(after, before); + assert_eq!(after[0].canonical_carrier_wire(), expected_wire); + assert_eq!(after[0].authentication_sidecar(), expected_sidecar); + } + + #[test] + fn outbound_cannot_be_exposed_before_both_exact_records_are_durable() { + let mut journal = journal(); + let own_candidate = candidate(1, 1, 0x51, Vec::new(), None); + let own = own_candidate.reference(); + let context = journal.context; + let expose = || JournalEventV1::ExposeOutbound { + context, + reference: own, + }; + assert_eq!( + journal.append(expose()).unwrap_err(), + JournalErrorV1::OwnCarrierNotFixed(own) + ); + journal + .append(JournalEventV1::PersistOutboundContent { + context: journal.context, + candidate: own_candidate, + }) + .unwrap(); + journal.append(fix_event(&journal, own)).unwrap(); + journal + .append(JournalEventV1::LockEcho { + context: journal.context, + target: own, + }) + .unwrap(); + assert_eq!( + journal.append(expose()).unwrap_err(), + JournalErrorV1::OutboundSidecarNotPersisted(own) + ); + } + + #[test] + fn phase_application_is_durable_before_the_cursor_advances() { + let mut journal = journal(); + let statement = RbcPhaseStatementV1::Echo { + target: reference(0, 1, 0x60), + }; + let outer_candidate = candidate(2, 3, 0x62, vec![statement], None); + let outer = outer_candidate.reference(); + admit_candidate(&mut journal, &outer_candidate); + let apply = JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer, + index: 0, + sender: 2, + statement, + }; + assert_before_after(&journal, apply.clone(), |state| { + state.phase_statement_applied(outer, 0) && state.phase_batch_cursor(outer) == 0 + }); + journal.append(apply.clone()).unwrap(); + + let advance = JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer, + index: 0, + }; + assert_before_after(&journal, advance, |state| { + state.phase_batch_cursor(outer) == 1 + }); + + let restarted = journal.restart().unwrap(); + let mut retried = restarted.clone(); + retried.append(apply).unwrap(); + retried + .append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer, + index: 0, + }) + .unwrap(); + assert_eq!(retried.snapshot().phase_batch_cursor(outer), 1); + } + + #[test] + fn only_admitted_conflict_processes_until_the_other_is_delivered() { + let mut journal = journal(); + let first_statement = RbcPhaseStatementV1::Echo { + target: reference(0, 1, 0x63), + }; + let second_statement = RbcPhaseStatementV1::Echo { + target: reference(3, 1, 0x64), + }; + let first_candidate = candidate(2, 2, 0x65, vec![first_statement], None); + let second_candidate = candidate(2, 2, 0x66, vec![second_statement], None); + let first = first_candidate.reference(); + let second = second_candidate.reference(); + authenticate_candidate(&mut journal, &first_candidate); + authenticate_candidate(&mut journal, &second_candidate); + journal + .append(JournalEventV1::LockAdmission { + context: journal.context, + target: first, + }) + .unwrap(); + assert!(matches!( + journal.append(JournalEventV1::LockAdmission { + context: journal.context, + target: second, + }), + Err(JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Admission, + .. + }) + )); + journal + .append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: first, + index: 0, + sender: 2, + statement: first_statement, + }) + .unwrap(); + assert_eq!( + journal + .append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: second, + index: 0, + sender: 2, + statement: second_statement, + }) + .unwrap_err(), + JournalErrorV1::OuterCarrierNotAdmittedOrDelivered(second) + ); + + journal + .append(JournalEventV1::LockReady { + context: journal.context, + target: second, + }) + .unwrap(); + journal + .append(JournalEventV1::LockDelivery { + context: journal.context, + target: second, + }) + .unwrap(); + journal + .append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: second, + index: 0, + sender: 2, + statement: second_statement, + }) + .unwrap(); + journal + .append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer: second, + index: 0, + }) + .unwrap(); + assert_eq!(journal.snapshot().phase_batch_cursor(second), 1); + } + + #[test] + fn cursor_cannot_skip_or_advance_before_idempotent_application() { + let mut journal = journal(); + let statement = RbcPhaseStatementV1::Ready { + target: reference(0, 1, 0x70), + }; + let second_statement = RbcPhaseStatementV1::Ready { + target: reference(3, 1, 0x71), + }; + let outer_candidate = candidate(2, 3, 0x72, vec![statement, second_statement], None); + let outer = outer_candidate.reference(); + admit_candidate(&mut journal, &outer_candidate); + assert!(matches!( + journal.append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer, + index: 0, + }), + Err(JournalErrorV1::PhaseCursorBeforeApplication { .. }) + )); + assert!(matches!( + journal.append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer, + index: 1, + sender: 2, + statement: second_statement, + }), + Err(JournalErrorV1::PhaseBatchIndexGap { + expected: 0, + actual: 1, + .. + }) + )); + } + + #[test] + fn own_embedded_phase_requires_lock_to_precede_it_in_the_log() { + let mut journal = journal(); + let target_candidate = candidate(0, 1, 0x80, Vec::new(), None); + let target = target_candidate.reference(); + let statement = RbcPhaseStatementV1::Echo { target }; + let outer_candidate = candidate(1, 2, 0x81, vec![statement], None); + let outer = outer_candidate.reference(); + admit_candidate(&mut journal, &target_candidate); + persist_and_fix(&mut journal, &outer_candidate); + let apply = JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer, + index: 0, + sender: 1, + statement, + }; + assert_eq!( + journal.append(apply.clone()).unwrap_err(), + JournalErrorV1::OwnPhaseWithoutDurableLock(statement) + ); + journal + .append(JournalEventV1::LockEcho { + context: journal.context, + target, + }) + .unwrap(); + journal.append(apply).unwrap(); + + let mut reversed = journal.durable_events().to_vec(); + let last = reversed.len() - 1; + reversed.swap(last - 1, last); + assert_eq!( + WriteAheadJournalV1::from_durable_events(journal.context, 1, reversed).unwrap_err(), + JournalErrorV1::OwnPhaseWithoutDurableLock(statement) + ); + } + + #[test] + fn conflicting_local_carrier_and_phase_choices_are_rejected() { + let mut journal = journal(); + let first_candidate = candidate(1, 1, 0x91, Vec::new(), None); + let second_candidate = candidate(1, 1, 0x92, Vec::new(), None); + let first_carrier = first_candidate.reference(); + let second_carrier = second_candidate.reference(); + journal + .append(outbound_content_event(&journal, &first_candidate)) + .unwrap(); + journal + .append(outbound_content_event(&journal, &second_candidate)) + .unwrap(); + journal.append(fix_event(&journal, first_carrier)).unwrap(); + assert_eq!( + journal + .append(fix_event(&journal, second_carrier)) + .unwrap_err(), + JournalErrorV1::ConflictingOwnCarrier(1) + ); + + let first_target_candidate = candidate(0, 1, 0x93, Vec::new(), None); + let second_target_candidate = candidate(0, 1, 0x94, Vec::new(), None); + let first_target = first_target_candidate.reference(); + let second_target = second_target_candidate.reference(); + retain_candidate(&mut journal, &first_target_candidate); + retain_candidate(&mut journal, &second_target_candidate); + journal + .append(JournalEventV1::LockReady { + context: journal.context, + target: first_target, + }) + .unwrap(); + assert!(matches!( + journal.append(JournalEventV1::LockReady { + context: journal.context, + target: second_target, + }), + Err(JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Ready, + .. + }) + )); + } + + #[test] + fn conflicting_remote_phase_is_durably_ignored_and_does_not_stall_cursor() { + let mut journal = journal(); + let first = reference(0, 1, 0xA0); + let second = reference(0, 1, 0xA1); + let first_statement = RbcPhaseStatementV1::Ready { target: first }; + let second_statement = RbcPhaseStatementV1::Ready { target: second }; + let outer_one_candidate = candidate(2, 2, 0xA2, vec![first_statement], None); + let outer_two_candidate = candidate(2, 3, 0xA3, vec![second_statement], None); + let outer_one = outer_one_candidate.reference(); + let outer_two = outer_two_candidate.reference(); + admit_candidate(&mut journal, &outer_one_candidate); + admit_candidate(&mut journal, &outer_two_candidate); + journal + .append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: outer_one, + index: 0, + sender: 2, + statement: first_statement, + }) + .unwrap(); + journal + .append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer: outer_one, + index: 0, + }) + .unwrap(); + journal + .append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: outer_two, + index: 0, + sender: 2, + statement: second_statement, + }) + .unwrap(); + assert_eq!( + journal.snapshot().phase_statement_outcome(outer_two, 0), + Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) + ); + journal + .append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer: outer_two, + index: 0, + }) + .unwrap(); + assert_eq!(journal.snapshot().phase_batch_cursor(outer_two), 1); + assert_eq!( + journal + .restart() + .unwrap() + .snapshot() + .phase_statement_outcome(outer_two, 0), + Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) + ); + } + + #[test] + fn replay_is_idempotent_and_foreign_namespace_fails_closed() { + let mut journal = journal(); + let own_candidate = candidate(1, 1, 0xB1, Vec::new(), None); + let own = own_candidate.reference(); + persist_and_fix(&mut journal, &own_candidate); + journal + .append(JournalEventV1::LockEcho { + context: journal.context, + target: own, + }) + .unwrap(); + let once = journal.restart().unwrap(); + let twice = once.restart().unwrap(); + assert_eq!(once.snapshot(), twice.snapshot()); + assert_eq!(once.durable_events(), twice.durable_events()); + + let foreign = JournalEventV1::LockReady { + context: context(0xB2), + target: own, + }; + assert!(matches!( + journal.append(foreign), + Err(JournalErrorV1::ContextMismatch) + )); + } + + #[test] + fn recovered_content_is_durable_before_ready_but_does_not_authorize_echo() { + let mut journal = journal(); + let target_candidate = candidate(0, 1, 0xB3, Vec::new(), None); + let target = target_candidate.reference(); + let ready = JournalEventV1::LockReady { + context: journal.context, + target, + }; + assert_eq!( + journal.append(ready.clone()).unwrap_err(), + JournalErrorV1::CarrierContentNotRetained(target) + ); + + let retain = JournalEventV1::RetainCandidateContent { + context: journal.context, + candidate: target_candidate.clone(), + }; + let expected_wire = target_candidate.canonical_wire_bytes().unwrap(); + assert_before_after(&journal, retain.clone(), |state| { + state.retained_carrier(target) == Some(expected_wire.as_slice()) + }); + journal.append(retain).unwrap(); + assert_before_after(&journal, ready.clone(), |state| { + state.ready_lock(RbcSlotKeyV1::of(target)) == Some(target) + }); + journal.append(ready).unwrap(); + + let echo = JournalEventV1::LockEcho { + context: journal.context, + target, + }; + assert_eq!( + journal.append(echo.clone()).unwrap_err(), + JournalErrorV1::EchoWithoutAdmission(target) + ); + admit_candidate(&mut journal, &target_candidate); + journal.append(echo).unwrap(); + } + + #[test] + fn own_fix_and_echo_cannot_precede_exact_typed_content() { + let mut journal = journal(); + let own_candidate = candidate(1, 1, 0xB4, Vec::new(), None); + let own = own_candidate.reference(); + assert_eq!( + journal.append(fix_event(&journal, own)).unwrap_err(), + JournalErrorV1::OutboundContentNotPersisted(own) + ); + + let content = outbound_content_event(&journal, &own_candidate); + let expected_wire = own_candidate.canonical_wire_bytes().unwrap(); + assert_before_after(&journal, content.clone(), |state| { + state.retained_carrier(own) == Some(expected_wire.as_slice()) + && state.own_carrier(1).is_none() + }); + journal.append(content).unwrap(); + let echo = JournalEventV1::LockEcho { + context: journal.context, + target: own, + }; + assert_eq!( + journal.append(echo.clone()).unwrap_err(), + JournalErrorV1::EchoWithoutAdmission(own) + ); + journal.append(fix_event(&journal, own)).unwrap(); + journal.append(echo).unwrap(); + } + + #[test] + fn outbound_exposure_waits_for_every_embedded_phase_lock() { + let mut journal = journal(); + let echo_target_candidate = candidate(0, 1, 0xB5, Vec::new(), None); + let ready_target_candidate = candidate(2, 1, 0xB6, Vec::new(), None); + let echo = RbcPhaseStatementV1::Echo { + target: echo_target_candidate.reference(), + }; + let ready = RbcPhaseStatementV1::Ready { + target: ready_target_candidate.reference(), + }; + let own_candidate = candidate(1, 2, 0xB7, vec![echo, ready], None); + let own = own_candidate.reference(); + admit_candidate(&mut journal, &echo_target_candidate); + retain_candidate(&mut journal, &ready_target_candidate); + prepare_outbound_for_exposure(&mut journal, &own_candidate); + let expose = JournalEventV1::ExposeOutbound { + context: journal.context, + reference: own, + }; + + assert_eq!( + journal.append(expose.clone()).unwrap_err(), + JournalErrorV1::OutboundPhaseNotLocked(echo) + ); + journal + .append(JournalEventV1::LockEcho { + context: journal.context, + target: echo.target(), + }) + .unwrap(); + assert_eq!( + journal.append(expose.clone()).unwrap_err(), + JournalErrorV1::OutboundPhaseNotLocked(ready) + ); + journal + .append(JournalEventV1::LockReady { + context: journal.context, + target: ready.target(), + }) + .unwrap(); + journal.append(expose).unwrap(); + } + + #[test] + fn outbound_exposure_waits_for_matching_consensus_and_leader_locks() { + let mut journal = journal(); + let vertex = consensus_vertex(1, 1); + let choice = vertex.leader_choice(); + let own_candidate = candidate(1, 2, 0xB8, Vec::new(), Some(vertex)); + let own = own_candidate.reference(); + prepare_outbound_for_exposure(&mut journal, &own_candidate); + let expose = JournalEventV1::ExposeOutbound { + context: journal.context, + reference: own, + }; + assert_eq!( + journal.append(expose.clone()).unwrap_err(), + JournalErrorV1::OutboundConsensusSlotNotLocked { + consensus_round: 1, + reference: own, + } + ); + journal + .append(JournalEventV1::LockConsensusSlot { + context: journal.context, + consensus_round: 1, + enclosing_carrier: own, + }) + .unwrap(); + assert_eq!( + journal.append(expose.clone()).unwrap_err(), + JournalErrorV1::OutboundLeaderChoiceNotLocked(1) + ); + journal + .append(JournalEventV1::LockLeaderChoice { + context: journal.context, + consensus_round: 1, + choice, + }) + .unwrap(); + journal.append(expose).unwrap(); + } + + #[test] + fn mismatched_consensus_or_leader_lock_cannot_poison_a_slot() { + let mut no_vertex_journal = journal(); + let no_vertex = candidate(1, 2, 0xB9, Vec::new(), None); + prepare_outbound_for_exposure(&mut no_vertex_journal, &no_vertex); + assert!(matches!( + no_vertex_journal.append(JournalEventV1::LockConsensusSlot { + context: no_vertex_journal.context, + consensus_round: 1, + enclosing_carrier: no_vertex.reference(), + }), + Err(JournalErrorV1::ConsensusVertexMismatch { .. }) + )); + + let mut journal = journal(); + let vertex = consensus_vertex(1, 1); + let choice = vertex.leader_choice(); + let own_candidate = candidate(1, 2, 0xBA, Vec::new(), Some(vertex)); + prepare_outbound_for_exposure(&mut journal, &own_candidate); + assert_eq!( + journal + .append(JournalEventV1::LockLeaderChoice { + context: journal.context, + consensus_round: 1, + choice, + }) + .unwrap_err(), + JournalErrorV1::LeaderChoiceWithoutConsensusSlot(1) + ); + assert!(matches!( + journal.append(JournalEventV1::LockConsensusSlot { + context: journal.context, + consensus_round: 2, + enclosing_carrier: own_candidate.reference(), + }), + Err(JournalErrorV1::ConsensusVertexMismatch { .. }) + )); + journal + .append(JournalEventV1::LockConsensusSlot { + context: journal.context, + consensus_round: 1, + enclosing_carrier: own_candidate.reference(), + }) + .unwrap(); + let wrong_choice = LeaderChoiceV1::NoVote { + leader_author: 3, + leader_round: 0, + }; + assert_eq!( + journal + .append(JournalEventV1::LockLeaderChoice { + context: journal.context, + consensus_round: 1, + choice: wrong_choice, + }) + .unwrap_err(), + JournalErrorV1::LeaderChoiceCandidateMismatch(1) + ); + journal + .append(JournalEventV1::LockLeaderChoice { + context: journal.context, + consensus_round: 1, + choice, + }) + .unwrap(); + } + + #[test] + fn authenticated_ingress_capability_is_context_and_receiver_bound() { + let candidate = candidate(0, 1, 0xBB, Vec::new(), None); + let mut journal = journal(); + assert!(matches!( + journal.append(JournalEventV1::AuthenticatedIngress { + context: journal.context, + sequence: 0, + authenticated: authenticated(&candidate, 2), + provenance: IngressProvenanceV1::Relayed { peer: 3 }, + }), + Err(JournalErrorV1::AuthenticatedIngressReceiverMismatch { + expected: 1, + actual: 2, + }) + )); + assert_eq!(journal.snapshot().next_ingress_sequence(), 0); + assert_eq!( + journal + .append(JournalEventV1::AuthenticatedIngress { + context: journal.context, + sequence: 0, + authenticated: authenticated_with_marker(&candidate, 1, 0xBC), + provenance: IngressProvenanceV1::DirectFromAuthor, + }) + .unwrap_err(), + JournalErrorV1::AuthenticatedIngressContextMismatch + ); + assert_eq!(journal.snapshot().next_ingress_sequence(), 0); + } + + #[test] + fn typed_outbound_content_rederives_bytes_and_rejects_foreign_context_sidecar() { + let mut journal = journal(); + let own_candidate = candidate(1, 1, 0xC1, Vec::new(), None); + let own = own_candidate.reference(); + let expected_wire = own_candidate.canonical_wire_bytes().unwrap(); + journal + .append(JournalEventV1::PersistOutboundContent { + context: journal.context, + candidate: own_candidate.clone(), + }) + .unwrap(); + assert_eq!( + journal.snapshot().retained_carrier(own), + Some(expected_wire.as_slice()) + ); + journal.append(fix_event(&journal, own)).unwrap(); + journal + .append(JournalEventV1::PersistOutboundSidecar { + context: journal.context, + authenticated: locally_authenticated(&own_candidate), + }) + .unwrap(); + assert_eq!( + journal + .append(JournalEventV1::PersistOutboundSidecar { + context: journal.context, + authenticated: locally_authenticated_with_marker(&own_candidate, 0xC2), + }) + .unwrap_err(), + JournalErrorV1::OutboundAuthenticationContextMismatch + ); + } +} diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs new file mode 100644 index 00000000..2a3afd17 --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -0,0 +1,2516 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Canonical carrier types for the experimental embedded-RBC Starfish DAG. +//! +//! This module is deliberately independent from the implemented direct-message +//! `starfish_rbc` protocol. Runtime and consensus integration are later +//! milestones. + +pub mod journal; +pub mod model; +pub mod projection; + +use std::{ + collections::{BTreeSet, HashSet}, + error::Error, + fmt, + sync::Arc, +}; + +use crate::{ + committee::Committee, + crypto::{ + Blake3Hasher, MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, MacKey, + MacTag, MlDsa44SignatureBytes, MlDsa44Signer, MlDsa65SignatureBytes, MlDsa65Signer, + SIGNATURE_SIZE, SignatureBytes, Signer, TransactionsCommitment, + }, + types::{ + AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, + RoundNumber, TimestampNs, + }, +}; + +pub const CARRIER_FORMAT_VERSION_V1: u8 = 1; +pub const CARRIER_WIRE_FORMAT_VERSION_V1: u8 = 0x81; +pub const MAX_CARRIER_CONTENT_SIZE_V1: usize = 4 * 1024 * 1024; +pub const MAX_PHASE_STATEMENTS_V1: usize = 2_048; + +const CONTENT_FORMAT_FIELD: u8 = 0x00; +const AUTHOR_FIELD: u8 = 0x01; +const CARRIER_ROUND_FIELD: u8 = 0x02; +const OWN_PREV_FIELD: u8 = 0x03; +const WEAK_PARENTS_FIELD: u8 = 0x04; +const TRANSACTIONS_COMMITMENT_FIELD: u8 = 0x05; +const ACKNOWLEDGMENTS_FIELD: u8 = 0x06; +const PHASE_BATCH_FIELD: u8 = 0x07; +const CONSENSUS_VERTEX_FIELD: u8 = 0x08; +const CREATION_TIME_FIELD: u8 = 0x09; +const CONSENSUS_ROUND_FIELD: u8 = 0x01; +const STRONG_PARENTS_FIELD: u8 = 0x02; +const DELIVERY_FRONTIER_FIELD: u8 = 0x03; +const LEADER_CHOICE_FIELD: u8 = 0x04; + +const OPTION_NONE: u8 = 0; +const OPTION_SOME: u8 = 1; +const PHASE_ECHO: u8 = 0; +const PHASE_READY: u8 = 1; +const LEADER_NONE: u8 = 0; +const LEADER_VOTE: u8 = 1; +const LEADER_NO_VOTE: u8 = 2; +const BLOCK_REFERENCE_SIZE: usize = 2 + 4 + 32; +const PROTOCOL_INSTANCE_SIZE: usize = 32; +const COMMITTEE_ID_SIZE: usize = 32; +const AUTHENTICATION_DOMAIN: &[u8; 19] = b"STARFISH_RBC_DAG_V1"; +const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_DAG_V1_COMMITTEE_ID"; +const CARRIER_AUTHENTICATION_KIND: u8 = 0; +const AUTHENTICATION_BASE_SIZE: usize = 123; +const AUTHENTICATION_MAC_SIZE: usize = AUTHENTICATION_BASE_SIZE + 2; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum RbcPhaseStatementV1 { + Echo { target: BlockReference }, + Ready { target: BlockReference }, +} + +impl RbcPhaseStatementV1 { + pub fn target(self) -> BlockReference { + match self { + Self::Echo { target } | Self::Ready { target } => target, + } + } + + fn code(self) -> u8 { + match self { + Self::Echo { .. } => PHASE_ECHO, + Self::Ready { .. } => PHASE_READY, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct ConsensusVertexReference { + carrier: BlockReference, + consensus_round: RoundNumber, +} + +impl ConsensusVertexReference { + pub const fn new(carrier: BlockReference, consensus_round: RoundNumber) -> Self { + Self { + carrier, + consensus_round, + } + } + + pub const fn carrier(self) -> BlockReference { + self.carrier + } + + pub const fn consensus_round(self) -> RoundNumber { + self.consensus_round + } + + pub const fn author(self) -> AuthorityIndex { + self.carrier.authority + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum LeaderChoiceV1 { + Vote { + leader: ConsensusVertexReference, + }, + NoVote { + leader_author: AuthorityIndex, + leader_round: RoundNumber, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConsensusVertexV1 { + consensus_round: RoundNumber, + strong_parents: Vec, + delivery_frontier: Vec>, + leader_choice: LeaderChoiceV1, +} + +impl ConsensusVertexV1 { + pub fn new( + consensus_round: RoundNumber, + strong_parents: Vec, + delivery_frontier: Vec>, + leader_choice: LeaderChoiceV1, + ) -> Self { + Self { + consensus_round, + strong_parents, + delivery_frontier, + leader_choice, + } + } + + pub fn consensus_round(&self) -> RoundNumber { + self.consensus_round + } + + pub fn strong_parents(&self) -> &[ConsensusVertexReference] { + &self.strong_parents + } + + pub fn delivery_frontier(&self) -> &[Option] { + &self.delivery_frontier + } + + pub fn leader_choice(&self) -> LeaderChoiceV1 { + self.leader_choice + } + + /// Validate the context-free certified-projection shape of this optional + /// vertex. Callers intentionally invoke this separately from carrier + /// candidacy: failure excludes only the optional vertex. + pub fn validate_projection_shape( + &self, + enclosing_author: AuthorityIndex, + committee: &Committee, + ) -> Result<(), RbcDagProjectionError> { + if self.consensus_round == 0 { + return Err(RbcDagProjectionError::GenesisVertexEncoded); + } + if !committee.known_authority(enclosing_author) { + return Err(RbcDagProjectionError::UnknownAuthority(enclosing_author)); + } + if self.strong_parents.len() > committee.len() { + return Err(RbcDagProjectionError::InvalidStrongParentCount( + self.strong_parents.len(), + )); + } + + let parent_round = self.consensus_round - 1; + let mut previous_authority = None; + let mut parent_stake = 0u64; + let mut includes_own_previous = false; + for parent in &self.strong_parents { + let authority = parent.author(); + if !committee.known_authority(authority) { + return Err(RbcDagProjectionError::UnknownAuthority(authority)); + } + if previous_authority.is_some_and(|previous| previous >= authority) { + return Err(RbcDagProjectionError::StrongParentsNotOrdered); + } + previous_authority = Some(authority); + if parent.consensus_round != parent_round { + return Err(RbcDagProjectionError::InvalidStrongParent(*parent)); + } + if parent_round == 0 && parent.carrier != carrier_genesis_reference(authority) { + return Err(RbcDagProjectionError::InvalidStrongParent(*parent)); + } + includes_own_previous |= authority == enclosing_author; + parent_stake = parent_stake + .checked_add( + committee + .get_stake(authority) + .ok_or(RbcDagProjectionError::UnknownAuthority(authority))?, + ) + .ok_or(RbcDagProjectionError::StakeOverflow)?; + } + if parent_stake < committee.quorum_threshold() { + return Err(RbcDagProjectionError::InvalidStrongParentThreshold); + } + if !includes_own_previous { + return Err(RbcDagProjectionError::MissingOwnStrongParent); + } + + if self.delivery_frontier.len() != committee.len() { + return Err(RbcDagProjectionError::InvalidFrontierLength { + expected: committee.len(), + actual: self.delivery_frontier.len(), + }); + } + for (authority, entry) in self.delivery_frontier.iter().enumerate() { + if let Some(reference) = entry { + if reference.authority as usize != authority || reference.round == 0 { + return Err(RbcDagProjectionError::InvalidFrontierEntry { + authority: authority as AuthorityIndex, + reference: *reference, + }); + } + } + } + + let expected_leader = committee.elect_leader(parent_round); + match self.leader_choice { + LeaderChoiceV1::Vote { leader } => { + if leader.consensus_round != parent_round + || leader.author() != expected_leader + || !self.strong_parents.contains(&leader) + { + return Err(RbcDagProjectionError::InvalidLeaderVote(leader)); + } + } + LeaderChoiceV1::NoVote { + leader_author, + leader_round, + } => { + if leader_author != expected_leader + || leader_round != parent_round + || self + .strong_parents + .iter() + .any(|parent| parent.author() == expected_leader) + { + return Err(RbcDagProjectionError::InvalidNoVote { + leader_author, + leader_round, + }); + } + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RbcDagProjectionError { + CommitteeMismatch, + GenesisVertexEncoded, + UnknownAuthority(AuthorityIndex), + InvalidStrongParentCount(usize), + StrongParentsNotOrdered, + InvalidStrongParent(ConsensusVertexReference), + StakeOverflow, + InvalidStrongParentThreshold, + MissingOwnStrongParent, + InvalidFrontierLength { + expected: usize, + actual: usize, + }, + InvalidFrontierEntry { + authority: AuthorityIndex, + reference: BlockReference, + }, + InvalidLeaderVote(ConsensusVertexReference), + InvalidNoVote { + leader_author: AuthorityIndex, + leader_round: RoundNumber, + }, +} + +impl fmt::Display for RbcDagProjectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "Starfish-RBC-DAG projection error: {self:?}") + } +} + +impl Error for RbcDagProjectionError {} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CarrierHeaderV1 { + author: AuthorityIndex, + carrier_round: RoundNumber, + own_prev: BlockReference, + weak_parents: Vec, + transactions_commitment: TransactionsCommitment, + data_acknowledgments: Vec, + phase_batch: Vec, + consensus_vertex: Option, + creation_time_ns: TimestampNs, +} + +#[derive(Clone, Debug)] +pub struct CarrierHeaderV1Args { + pub author: AuthorityIndex, + pub carrier_round: RoundNumber, + pub own_prev: BlockReference, + pub weak_parents: Vec, + pub transactions_commitment: TransactionsCommitment, + pub data_acknowledgments: Vec, + pub phase_batch: Vec, + pub consensus_vertex: Option, + pub creation_time_ns: TimestampNs, +} + +impl CarrierHeaderV1 { + fn from_args(args: CarrierHeaderV1Args) -> Self { + Self { + author: args.author, + carrier_round: args.carrier_round, + own_prev: args.own_prev, + weak_parents: args.weak_parents, + transactions_commitment: args.transactions_commitment, + data_acknowledgments: args.data_acknowledgments, + phase_batch: args.phase_batch, + consensus_vertex: args.consensus_vertex, + creation_time_ns: args.creation_time_ns, + } + } +} + +impl CarrierHeaderV1 { + pub fn author(&self) -> AuthorityIndex { + self.author + } + + pub fn carrier_round(&self) -> RoundNumber { + self.carrier_round + } + + pub fn own_prev(&self) -> BlockReference { + self.own_prev + } + + pub fn weak_parents(&self) -> &[BlockReference] { + &self.weak_parents + } + + pub fn transactions_commitment(&self) -> TransactionsCommitment { + self.transactions_commitment + } + + pub fn data_acknowledgments(&self) -> &[BlockReference] { + &self.data_acknowledgments + } + + pub fn phase_batch(&self) -> &[RbcPhaseStatementV1] { + &self.phase_batch + } + + pub fn consensus_vertex(&self) -> Option<&ConsensusVertexV1> { + self.consensus_vertex.as_ref() + } + + pub fn creation_time_ns(&self) -> TimestampNs { + self.creation_time_ns + } + + /// Exact expanded canonical content bytes committed by the carrier + /// reference. Authentication is deliberately absent. + fn canonical_content_bytes(&self) -> Result, RbcDagError> { + encode_header(self, AckEncoding::Expanded) + } + + /// Canonical wire bytes. The acknowledgment log is represented by its + /// unique maximal physical-parent suffix and the remaining exact log. + fn canonical_wire_bytes(&self) -> Result, RbcDagError> { + encode_header(self, AckEncoding::Compressed) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CandidateCarrierV1 { + header: Arc, + reference: BlockReference, + committee_id: RbcDagCommitteeId, +} + +impl CandidateCarrierV1 { + pub fn try_new(args: CarrierHeaderV1Args, committee: &Committee) -> Result { + Self::try_from_header(CarrierHeaderV1::from_args(args), committee, None) + } + + pub fn try_from_header( + mut header: CarrierHeaderV1, + committee: &Committee, + expected_reference: Option, + ) -> Result { + normalize_acknowledgments(&mut header)?; + validate_outer_header(&header, committee)?; + let reference = carrier_reference(&header)?; + let committee_id = RbcDagCommitteeId::derive(committee)?; + if let Some(expected) = expected_reference { + if expected != reference { + return Err(RbcDagError::ReferenceMismatch { + expected, + actual: reference, + }); + } + } + Ok(Self { + header: Arc::new(header), + reference, + committee_id, + }) + } + + pub fn decode_content( + bytes: &[u8], + committee: &Committee, + expected_reference: Option, + ) -> Result { + let header = decode_header(bytes, AckEncoding::Expanded)?; + let candidate = Self::try_from_header(header, committee, expected_reference)?; + if candidate.canonical_content_bytes()?.as_slice() != bytes { + return Err(RbcDagError::NonCanonicalAcknowledgments); + } + Ok(candidate) + } + + pub fn decode_wire( + bytes: &[u8], + committee: &Committee, + expected_reference: Option, + ) -> Result { + let header = decode_header(bytes, AckEncoding::Compressed)?; + Self::try_from_header(header, committee, expected_reference) + } + + pub fn header(&self) -> &CarrierHeaderV1 { + &self.header + } + + pub fn reference(&self) -> BlockReference { + self.reference + } + + pub fn committee_id(&self) -> RbcDagCommitteeId { + self.committee_id + } + + pub fn canonical_content_bytes(&self) -> Result, RbcDagError> { + self.header.canonical_content_bytes() + } + + pub fn canonical_wire_bytes(&self) -> Result, RbcDagError> { + self.header.canonical_wire_bytes() + } + + pub fn validate_consensus_vertex( + &self, + committee: &Committee, + ) -> Result, RbcDagProjectionError> { + let committee_id = RbcDagCommitteeId::derive(committee) + .map_err(|_| RbcDagProjectionError::CommitteeMismatch)?; + if committee_id != self.committee_id { + return Err(RbcDagProjectionError::CommitteeMismatch); + } + let Some(vertex) = self.header.consensus_vertex() else { + return Ok(None); + }; + vertex.validate_projection_shape(self.header.author(), committee)?; + Ok(Some(vertex)) + } +} + +#[derive(Clone, Eq, PartialEq)] +pub struct FlatMacVector { + bytes: Box<[u8]>, +} + +impl FlatMacVector { + pub fn from_tags(tags: &[MacTag]) -> Result { + if tags.len() > MAX_COMMITTEE_SIZE as usize { + return Err(RbcDagError::InvalidMacVectorLength { + expected: MAX_COMMITTEE_SIZE as usize * MAC_TAG_SIZE, + actual: tags.len().saturating_mul(MAC_TAG_SIZE), + }); + } + let mut bytes = Vec::with_capacity(tags.len() * MAC_TAG_SIZE); + for tag in tags { + bytes.extend_from_slice(tag.as_ref()); + } + Ok(Self { + bytes: bytes.into_boxed_slice(), + }) + } + + pub fn from_bytes(bytes: Vec) -> Result { + if !bytes.chunks_exact(MAC_TAG_SIZE).remainder().is_empty() + || bytes.len() > MAX_COMMITTEE_SIZE as usize * MAC_TAG_SIZE + { + return Err(RbcDagError::InvalidFlatMacVectorLength(bytes.len())); + } + Ok(Self { + bytes: bytes.into_boxed_slice(), + }) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn len(&self) -> usize { + self.bytes.len() / MAC_TAG_SIZE + } + + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + pub fn tag(&self, authority: AuthorityIndex) -> Option { + let start = authority as usize * MAC_TAG_SIZE; + let end = start.checked_add(MAC_TAG_SIZE)?; + let mut bytes = [0; MAC_TAG_SIZE]; + bytes.copy_from_slice(self.bytes.get(start..end)?); + Some(MacTag::from_bytes(bytes)) + } +} + +impl fmt::Debug for FlatMacVector { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FlatMacVector") + .field("tag_count", &self.len()) + .finish() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CarrierAuthenticationV1 { + Ed25519(SignatureBytes), + MlDsa44(MlDsa44SignatureBytes), + MlDsa65(MlDsa65SignatureBytes), + MacVector(FlatMacVector), +} + +impl CarrierAuthenticationV1 { + pub fn scheme(&self) -> BlockAuthenticationScheme { + match self { + Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, + Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, + Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, + Self::MacVector(_) => BlockAuthenticationScheme::MacVector, + } + } + + /// Versioned sidecar wire bytes. `FlatMacVector` itself remains the raw + /// concatenation of tags; the envelope supplies version and scheme. + pub fn canonical_wire_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.push(CONTENT_FORMAT_FIELD); + bytes.push(CARRIER_FORMAT_VERSION_V1); + bytes.push(authentication_scheme_code(self.scheme())); + match self { + Self::Ed25519(signature) => bytes.extend_from_slice(signature.as_ref()), + Self::MlDsa44(signature) => bytes.extend_from_slice(signature.as_ref()), + Self::MlDsa65(signature) => bytes.extend_from_slice(signature.as_ref()), + Self::MacVector(vector) => bytes.extend_from_slice(vector.as_bytes()), + } + bytes + } + + pub fn decode_wire(bytes: &[u8], committee: &Committee) -> Result { + validate_committee(committee)?; + let mut decoder = Decoder::new(bytes); + decoder.expect_marker(CONTENT_FORMAT_FIELD)?; + let version = decoder.read_u8()?; + if version != CARRIER_FORMAT_VERSION_V1 { + return Err(RbcDagError::UnsupportedVersion(version)); + } + let scheme = decode_authentication_scheme(decoder.read_u8()?)?; + let authentication = match scheme { + BlockAuthenticationScheme::Ed25519 => Self::Ed25519(SignatureBytes::from_bytes( + decoder.read_array::()?, + )), + BlockAuthenticationScheme::MlDsa44 => Self::MlDsa44(MlDsa44SignatureBytes::from_bytes( + decoder.read_array::()?, + )), + BlockAuthenticationScheme::MlDsa65 => Self::MlDsa65(MlDsa65SignatureBytes::from_bytes( + decoder.read_array::()?, + )), + BlockAuthenticationScheme::MacVector => { + let expected = committee + .len() + .checked_mul(MAC_TAG_SIZE) + .ok_or(RbcDagError::InvalidCommittee("MAC vector length overflow"))?; + let vector = FlatMacVector::from_bytes(decoder.take(expected)?.to_vec())?; + Self::MacVector(vector) + } + }; + decoder.finish()?; + Ok(authentication) + } +} + +pub enum CarrierAuthorizerV1<'a> { + Ed25519 { + authority: AuthorityIndex, + signer: &'a Signer, + }, + MlDsa44 { + authority: AuthorityIndex, + signer: &'a MlDsa44Signer, + }, + MlDsa65 { + authority: AuthorityIndex, + signer: &'a MlDsa65Signer, + }, + MacVector { + authority: AuthorityIndex, + keys: &'a [MacKey], + }, +} + +impl CarrierAuthorizerV1<'_> { + fn scheme(&self) -> BlockAuthenticationScheme { + match self { + Self::Ed25519 { .. } => BlockAuthenticationScheme::Ed25519, + Self::MlDsa44 { .. } => BlockAuthenticationScheme::MlDsa44, + Self::MlDsa65 { .. } => BlockAuthenticationScheme::MlDsa65, + Self::MacVector { .. } => BlockAuthenticationScheme::MacVector, + } + } + + fn authority(&self) -> AuthorityIndex { + match self { + Self::Ed25519 { authority, .. } + | Self::MlDsa44 { authority, .. } + | Self::MlDsa65 { authority, .. } + | Self::MacVector { authority, .. } => *authority, + } + } +} + +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct RbcDagProtocolInstanceId([u8; PROTOCOL_INSTANCE_SIZE]); + +impl RbcDagProtocolInstanceId { + pub fn new(bytes: [u8; PROTOCOL_INSTANCE_SIZE]) -> Result { + if bytes.iter().all(|byte| *byte == 0) { + return Err(RbcDagError::ZeroProtocolInstance); + } + Ok(Self(bytes)) + } + + pub fn as_bytes(&self) -> &[u8; PROTOCOL_INSTANCE_SIZE] { + &self.0 + } +} + +impl fmt::Debug for RbcDagProtocolInstanceId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "RbcDagInstance({})", hex::encode(&self.0[..4])) + } +} + +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct RbcDagCommitteeId([u8; COMMITTEE_ID_SIZE]); + +impl RbcDagCommitteeId { + pub fn derive(committee: &Committee) -> Result { + validate_committee(committee)?; + let committee_size = u16::try_from(committee.len()) + .map_err(|_| RbcDagError::InvalidCommittee("committee too large"))?; + let info_length = u16::try_from(committee.info_length()) + .map_err(|_| RbcDagError::InvalidCommittee("information length too large"))?; + let mut hasher = Blake3Hasher::new_derive_key(COMMITTEE_ID_DERIVE_CONTEXT); + hasher.update(&committee_size.to_be_bytes()); + hasher.update(&committee.validity_threshold().to_be_bytes()); + hasher.update(&committee.quorum_threshold().to_be_bytes()); + hasher.update(&info_length.to_be_bytes()); + hasher.update(&committee.optimistic_fast_threshold().to_be_bytes()); + hasher.update(&committee.optimistic_vote_threshold().to_be_bytes()); + hasher.update(&committee.optimistic_ready_threshold().to_be_bytes()); + for authority in committee.authorities() { + let stake = committee + .get_stake(authority) + .ok_or(RbcDagError::UnknownAuthority(authority))?; + let public_key = committee + .get_public_key(authority) + .ok_or(RbcDagError::UnknownAuthority(authority))?; + let bls_public_key = committee + .get_bls_public_key(authority) + .ok_or(RbcDagError::UnknownAuthority(authority))?; + let ml_dsa_44_public_key = committee + .get_ml_dsa_44_public_key(authority) + .ok_or(RbcDagError::UnknownAuthority(authority))?; + let ml_dsa_65_public_key = committee + .get_ml_dsa_65_public_key(authority) + .ok_or(RbcDagError::UnknownAuthority(authority))?; + hasher.update(&authority.to_be_bytes()); + hasher.update(&stake.to_be_bytes()); + hasher.update(&public_key.to_bytes()); + hasher.update(&bls_public_key.to_bytes()); + hasher.update(&ml_dsa_44_public_key.to_bytes()); + hasher.update(&ml_dsa_65_public_key.to_bytes()); + } + Ok(Self(hasher.finalize().into())) + } + + pub fn as_bytes(&self) -> &[u8; COMMITTEE_ID_SIZE] { + &self.0 + } +} + +impl fmt::Debug for RbcDagCommitteeId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "RbcDagCommittee({})", hex::encode(&self.0[..4])) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RbcDagContextV1 { + protocol_instance: RbcDagProtocolInstanceId, + committee_id: RbcDagCommitteeId, + authentication_scheme: BlockAuthenticationScheme, +} + +impl RbcDagContextV1 { + pub fn new( + protocol_instance: RbcDagProtocolInstanceId, + committee: &Committee, + authentication_scheme: BlockAuthenticationScheme, + ) -> Result { + Ok(Self { + protocol_instance, + committee_id: RbcDagCommitteeId::derive(committee)?, + authentication_scheme, + }) + } + + pub fn protocol_instance(&self) -> RbcDagProtocolInstanceId { + self.protocol_instance + } + + pub fn committee_id(&self) -> RbcDagCommitteeId { + self.committee_id + } + + pub fn authentication_scheme(&self) -> BlockAuthenticationScheme { + self.authentication_scheme + } + + pub fn authenticate( + &self, + candidate: &CandidateCarrierV1, + committee: &Committee, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + self.ensure_committee(committee)?; + self.ensure_candidate(candidate)?; + if authorizer.scheme() != self.authentication_scheme { + return Err(RbcDagError::AuthenticationSchemeMismatch); + } + let reference = candidate.reference; + if authorizer.authority() != reference.authority { + return Err(RbcDagError::AuthorizerAuthorityMismatch { + expected: reference.authority, + actual: authorizer.authority(), + }); + } + match authorizer { + CarrierAuthorizerV1::Ed25519 { signer, .. } => { + let expected = committee + .get_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + if &signer.public_key() != expected { + return Err(RbcDagError::AuthorizerKeyMismatch); + } + Ok(CarrierAuthenticationV1::Ed25519(signer.sign_digest( + &self.public_authentication_digest(reference), + ))) + } + CarrierAuthorizerV1::MlDsa44 { signer, .. } => { + let expected = committee + .get_ml_dsa_44_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + if &signer.public_key() != expected { + return Err(RbcDagError::AuthorizerKeyMismatch); + } + let digest = BlockDigest::from(self.public_authentication_digest(reference)); + Ok(CarrierAuthenticationV1::MlDsa44( + signer.sign_digest(&digest), + )) + } + CarrierAuthorizerV1::MlDsa65 { signer, .. } => { + let expected = committee + .get_ml_dsa_65_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + if &signer.public_key() != expected { + return Err(RbcDagError::AuthorizerKeyMismatch); + } + let digest = BlockDigest::from(self.public_authentication_digest(reference)); + Ok(CarrierAuthenticationV1::MlDsa65( + signer.sign_digest(&digest), + )) + } + CarrierAuthorizerV1::MacVector { keys, .. } => { + if keys.len() != committee.len() { + return Err(RbcDagError::InvalidKeyringLength { + expected: committee.len(), + actual: keys.len(), + }); + } + let tags = committee + .authorities() + .map(|recipient| { + keys[recipient as usize].compute_rbc_tag( + &self.mac_authentication_statement(reference, recipient), + ) + }) + .collect::>(); + Ok(CarrierAuthenticationV1::MacVector( + FlatMacVector::from_tags(&tags)?, + )) + } + } + } + + /// Generate the exact authentication sidecar for a locally authored + /// carrier and bind it to the candidate and protocol context. + /// + /// The returned capability has private fields so persistence and network + /// adapters cannot substitute a freely constructed, same-scheme sidecar + /// for the one produced by the configured authorizer. + pub fn authenticate_local( + &self, + candidate: CandidateCarrierV1, + committee: &Committee, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + let authentication = self.authenticate(&candidate, committee, authorizer)?; + Ok(LocallyAuthenticatedCarrierV1 { + candidate, + authentication, + context: *self, + }) + } + + pub fn verify_authentication( + &self, + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + receiver: AuthorityIndex, + committee: &Committee, + mac_keys: &[MacKey], + ) -> Result { + self.ensure_committee(committee)?; + self.ensure_candidate(&candidate)?; + if !committee.known_authority(receiver) { + return Err(RbcDagError::UnknownAuthority(receiver)); + } + if authentication.scheme() != self.authentication_scheme { + return Err(RbcDagError::AuthenticationSchemeMismatch); + } + let reference = candidate.reference; + match &authentication { + CarrierAuthenticationV1::Ed25519(signature) => { + let public_key = committee + .get_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + public_key + .verify_digest_signature( + &self.public_authentication_digest(reference), + signature, + ) + .map_err(|_| RbcDagError::InvalidAuthentication)?; + } + CarrierAuthenticationV1::MlDsa44(signature) => { + let public_key = committee + .get_ml_dsa_44_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + let digest = BlockDigest::from(self.public_authentication_digest(reference)); + public_key + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcDagError::InvalidAuthentication)?; + } + CarrierAuthenticationV1::MlDsa65(signature) => { + let public_key = committee + .get_ml_dsa_65_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + let digest = BlockDigest::from(self.public_authentication_digest(reference)); + public_key + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcDagError::InvalidAuthentication)?; + } + CarrierAuthenticationV1::MacVector(vector) => { + let expected_length = committee.len() * MAC_TAG_SIZE; + if vector.as_bytes().len() != expected_length { + return Err(RbcDagError::InvalidMacVectorLength { + expected: expected_length, + actual: vector.as_bytes().len(), + }); + } + if mac_keys.len() != committee.len() { + return Err(RbcDagError::InvalidKeyringLength { + expected: committee.len(), + actual: mac_keys.len(), + }); + } + let key = mac_keys + .get(reference.authority as usize) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + let expected = + key.compute_rbc_tag(&self.mac_authentication_statement(reference, receiver)); + let actual = vector + .tag(receiver) + .ok_or(RbcDagError::InvalidAuthentication)?; + if actual != expected { + return Err(RbcDagError::InvalidAuthentication); + } + } + } + Ok(AuthenticatedCarrierV1 { + candidate, + authentication, + context: *self, + receiver, + }) + } + + pub fn public_authentication_statement( + &self, + reference: BlockReference, + ) -> [u8; AUTHENTICATION_BASE_SIZE] { + encode_authentication_base(self, reference) + } + + pub fn mac_authentication_statement( + &self, + reference: BlockReference, + recipient: AuthorityIndex, + ) -> [u8; AUTHENTICATION_MAC_SIZE] { + let mut statement = [0; AUTHENTICATION_MAC_SIZE]; + statement[..AUTHENTICATION_BASE_SIZE] + .copy_from_slice(&encode_authentication_base(self, reference)); + statement[AUTHENTICATION_BASE_SIZE..].copy_from_slice(&recipient.to_be_bytes()); + statement + } + + pub fn public_authentication_digest(&self, reference: BlockReference) -> [u8; 32] { + blake3::hash(&self.public_authentication_statement(reference)).into() + } + + fn ensure_committee(&self, committee: &Committee) -> Result<(), RbcDagError> { + let actual = RbcDagCommitteeId::derive(committee)?; + if actual != self.committee_id { + return Err(RbcDagError::CommitteeIdMismatch); + } + Ok(()) + } + + fn ensure_candidate(&self, candidate: &CandidateCarrierV1) -> Result<(), RbcDagError> { + if candidate.committee_id != self.committee_id { + return Err(RbcDagError::CandidateCommitteeMismatch); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AuthenticatedCarrierV1 { + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + context: RbcDagContextV1, + receiver: AuthorityIndex, +} + +/// Opaque proof that the configured local authorizer generated a carrier's +/// exact sidecar. This is distinct from [`AuthenticatedCarrierV1`], which +/// proves that one receiver verified an inbound sidecar. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocallyAuthenticatedCarrierV1 { + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + context: RbcDagContextV1, +} + +impl LocallyAuthenticatedCarrierV1 { + pub fn candidate(&self) -> &CandidateCarrierV1 { + &self.candidate + } + + pub fn reference(&self) -> BlockReference { + self.candidate.reference() + } + + pub fn authentication(&self) -> &CarrierAuthenticationV1 { + &self.authentication + } + + pub fn context(&self) -> RbcDagContextV1 { + self.context + } + + pub fn into_parts(self) -> (CandidateCarrierV1, CarrierAuthenticationV1, RbcDagContextV1) { + (self.candidate, self.authentication, self.context) + } +} + +impl AuthenticatedCarrierV1 { + pub fn candidate(&self) -> &CandidateCarrierV1 { + &self.candidate + } + + pub fn header(&self) -> &CarrierHeaderV1 { + self.candidate.header() + } + + pub fn reference(&self) -> BlockReference { + self.candidate.reference() + } + + pub fn authentication(&self) -> &CarrierAuthenticationV1 { + &self.authentication + } + + pub fn context(&self) -> RbcDagContextV1 { + self.context + } + + pub fn receiver(&self) -> AuthorityIndex { + self.receiver + } + + pub fn into_parts( + self, + ) -> ( + CandidateCarrierV1, + CarrierAuthenticationV1, + RbcDagContextV1, + AuthorityIndex, + ) { + ( + self.candidate, + self.authentication, + self.context, + self.receiver, + ) + } +} + +fn authentication_scheme_code(authentication_scheme: BlockAuthenticationScheme) -> u8 { + match authentication_scheme { + BlockAuthenticationScheme::Ed25519 => 0, + BlockAuthenticationScheme::MlDsa44 => 1, + BlockAuthenticationScheme::MlDsa65 => 2, + BlockAuthenticationScheme::MacVector => 3, + } +} + +fn decode_authentication_scheme(code: u8) -> Result { + match code { + 0 => Ok(BlockAuthenticationScheme::Ed25519), + 1 => Ok(BlockAuthenticationScheme::MlDsa44), + 2 => Ok(BlockAuthenticationScheme::MlDsa65), + 3 => Ok(BlockAuthenticationScheme::MacVector), + other => Err(RbcDagError::InvalidAuthenticationScheme(other)), + } +} + +fn encode_authentication_base( + context: &RbcDagContextV1, + reference: BlockReference, +) -> [u8; AUTHENTICATION_BASE_SIZE] { + let mut statement = [0; AUTHENTICATION_BASE_SIZE]; + statement[..19].copy_from_slice(AUTHENTICATION_DOMAIN); + statement[19] = CARRIER_AUTHENTICATION_KIND; + statement[20] = authentication_scheme_code(context.authentication_scheme); + statement[21..53].copy_from_slice(&context.protocol_instance.0); + statement[53..85].copy_from_slice(&context.committee_id.0); + statement[85..87].copy_from_slice(&reference.authority.to_be_bytes()); + statement[87..91].copy_from_slice(&reference.round.to_be_bytes()); + statement[91..123].copy_from_slice(reference.digest.as_ref()); + statement +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RbcDagError { + UnexpectedEnd, + TrailingBytes(usize), + UnsupportedVersion(u8), + InvalidMarker { + expected: u8, + actual: u8, + }, + InvalidOption(u8), + InvalidPhase(u8), + InvalidLeaderChoice(u8), + VectorTooLong { + field: &'static str, + count: usize, + }, + ContentTooLarge(usize), + NonCanonicalAcknowledgments, + InvalidCommittee(&'static str), + UnknownAuthority(AuthorityIndex), + GenesisCarrier, + InvalidOwnPrevious, + InvalidWeakParent(BlockReference), + WeakParentsNotOrdered, + InvalidCarrierThreshold, + InvalidAcknowledgment(BlockReference), + DuplicateAcknowledgment(BlockReference), + InvalidPhaseTarget(BlockReference), + DuplicatePhaseStatement(RbcPhaseStatementV1), + InvalidConsensusRound, + ZeroProtocolInstance, + CommitteeIdMismatch, + CandidateCommitteeMismatch, + AuthenticationSchemeMismatch, + InvalidAuthenticationScheme(u8), + AuthorizerAuthorityMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + AuthorizerKeyMismatch, + InvalidAuthentication, + InvalidFlatMacVectorLength(usize), + InvalidMacVectorLength { + expected: usize, + actual: usize, + }, + InvalidKeyringLength { + expected: usize, + actual: usize, + }, + ReferenceMismatch { + expected: BlockReference, + actual: BlockReference, + }, +} + +impl fmt::Display for RbcDagError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "Starfish-RBC-DAG error: {self:?}") + } +} + +impl Error for RbcDagError {} + +fn validate_outer_header( + header: &CarrierHeaderV1, + committee: &Committee, +) -> Result<(), RbcDagError> { + validate_committee(committee)?; + if header.carrier_round == 0 { + return Err(RbcDagError::GenesisCarrier); + } + if !committee.known_authority(header.author) { + return Err(RbcDagError::UnknownAuthority(header.author)); + } + if header.own_prev.authority != header.author + || header.own_prev.round.checked_add(1) != Some(header.carrier_round) + || (header.carrier_round == 1 + && header.own_prev != carrier_genesis_reference(header.author)) + { + return Err(RbcDagError::InvalidOwnPrevious); + } + + if header.weak_parents.len() >= committee.len() + || header.weak_parents.len() > MAX_COMMITTEE_SIZE as usize - 1 + { + return Err(RbcDagError::VectorTooLong { + field: "weak parents", + count: header.weak_parents.len(), + }); + } + let mut previous_authority = None; + let mut parent_stake = committee + .get_stake(header.author) + .ok_or(RbcDagError::UnknownAuthority(header.author))?; + for parent in &header.weak_parents { + if !committee.known_authority(parent.authority) { + return Err(RbcDagError::UnknownAuthority(parent.authority)); + } + if parent.authority == header.author + || parent.round.checked_add(1) != Some(header.carrier_round) + || (header.carrier_round == 1 && *parent != carrier_genesis_reference(parent.authority)) + { + return Err(RbcDagError::InvalidWeakParent(*parent)); + } + if previous_authority.is_some_and(|previous| previous >= parent.authority) { + return Err(RbcDagError::WeakParentsNotOrdered); + } + previous_authority = Some(parent.authority); + parent_stake = parent_stake + .checked_add( + committee + .get_stake(parent.authority) + .ok_or(RbcDagError::UnknownAuthority(parent.authority))?, + ) + .ok_or(RbcDagError::InvalidCommittee("stake overflow"))?; + } + if parent_stake < committee.quorum_threshold() { + return Err(RbcDagError::InvalidCarrierThreshold); + } + + if header.data_acknowledgments.len() > u16::MAX as usize { + return Err(RbcDagError::VectorTooLong { + field: "acknowledgments", + count: header.data_acknowledgments.len(), + }); + } + let mut acknowledgments = BTreeSet::new(); + for acknowledgment in &header.data_acknowledgments { + if !committee.known_authority(acknowledgment.authority) + || acknowledgment.round == 0 + || acknowledgment.round > header.carrier_round + { + return Err(RbcDagError::InvalidAcknowledgment(*acknowledgment)); + } + if !acknowledgments.insert(*acknowledgment) { + return Err(RbcDagError::DuplicateAcknowledgment(*acknowledgment)); + } + } + + let phase_limit = usize::min(MAX_PHASE_STATEMENTS_V1, committee.len().saturating_mul(4)); + if header.phase_batch.len() > phase_limit { + return Err(RbcDagError::VectorTooLong { + field: "phase statements", + count: header.phase_batch.len(), + }); + } + let mut phase_statements = HashSet::new(); + for statement in &header.phase_batch { + let target = statement.target(); + if !committee.known_authority(target.authority) + || target.round == 0 + || target.round >= header.carrier_round + { + return Err(RbcDagError::InvalidPhaseTarget(target)); + } + if !phase_statements.insert((statement.code(), target.authority, target.round)) { + return Err(RbcDagError::DuplicatePhaseStatement(*statement)); + } + } + + if let Some(vertex) = &header.consensus_vertex { + if vertex.consensus_round == 0 { + return Err(RbcDagError::InvalidConsensusRound); + } + for (field, count) in [ + ("strong parents", vertex.strong_parents.len()), + ("delivery frontier", vertex.delivery_frontier.len()), + ] { + if count > MAX_COMMITTEE_SIZE as usize { + return Err(RbcDagError::VectorTooLong { field, count }); + } + } + } + + let content_size = header.canonical_content_bytes()?.len(); + if content_size > MAX_CARRIER_CONTENT_SIZE_V1 { + return Err(RbcDagError::ContentTooLarge(content_size)); + } + // Candidacy guarantees that the same logical carrier has a canonical + // transport representation as well as an identity representation. + header.canonical_wire_bytes()?; + Ok(()) +} + +fn carrier_reference(header: &CarrierHeaderV1) -> Result { + let bytes = header.canonical_content_bytes()?; + Ok(BlockReference { + round: header.carrier_round, + authority: header.author, + digest: BlockDigest::from(*blake3::hash(&bytes).as_bytes()), + }) +} + +/// Fixed virtual carrier-genesis reference for one authority. +pub fn carrier_genesis_reference(authority: AuthorityIndex) -> BlockReference { + let digest = BlockDigest::new_without_transactions(authority, 0, &[], &[], 0, None, None); + BlockReference { + round: 0, + authority, + digest, + } +} + +#[derive(Clone, Copy)] +enum AckEncoding { + Expanded, + Compressed, +} + +fn encode_header( + header: &CarrierHeaderV1, + acknowledgment_encoding: AckEncoding, +) -> Result, RbcDagError> { + let mut bytes = Vec::new(); + bytes.push(CONTENT_FORMAT_FIELD); + bytes.push(match acknowledgment_encoding { + AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, + AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, + }); + bytes.push(AUTHOR_FIELD); + bytes.extend_from_slice(&header.author.to_be_bytes()); + bytes.push(CARRIER_ROUND_FIELD); + bytes.extend_from_slice(&header.carrier_round.to_be_bytes()); + bytes.push(OWN_PREV_FIELD); + encode_reference(&mut bytes, header.own_prev); + bytes.push(WEAK_PARENTS_FIELD); + encode_count(&mut bytes, "weak parents", header.weak_parents.len())?; + for parent in &header.weak_parents { + encode_reference(&mut bytes, *parent); + } + bytes.push(TRANSACTIONS_COMMITMENT_FIELD); + bytes.extend_from_slice(header.transactions_commitment.as_ref()); + bytes.push(ACKNOWLEDGMENTS_FIELD); + match acknowledgment_encoding { + AckEncoding::Expanded => { + encode_count( + &mut bytes, + "acknowledgments", + header.data_acknowledgments.len(), + )?; + for acknowledgment in &header.data_acknowledgments { + encode_reference(&mut bytes, *acknowledgment); + } + } + AckEncoding::Compressed => { + let (intersection, extras) = compressed_acknowledgments(header)?; + bytes.extend_from_slice(&intersection.to_be_bytes()); + encode_count(&mut bytes, "extra acknowledgments", extras.len())?; + for acknowledgment in &extras { + encode_reference(&mut bytes, *acknowledgment); + } + } + } + bytes.push(PHASE_BATCH_FIELD); + encode_count(&mut bytes, "phase statements", header.phase_batch.len())?; + for statement in &header.phase_batch { + bytes.push(statement.code()); + encode_reference(&mut bytes, statement.target()); + } + bytes.push(CONSENSUS_VERTEX_FIELD); + match &header.consensus_vertex { + None => bytes.push(OPTION_NONE), + Some(vertex) => { + bytes.push(OPTION_SOME); + encode_consensus_vertex(&mut bytes, vertex)?; + } + } + bytes.push(CREATION_TIME_FIELD); + bytes.extend_from_slice(&header.creation_time_ns.to_be_bytes()); + if bytes.len() > MAX_CARRIER_CONTENT_SIZE_V1 { + return Err(RbcDagError::ContentTooLarge(bytes.len())); + } + Ok(bytes) +} + +fn encode_consensus_vertex( + bytes: &mut Vec, + vertex: &ConsensusVertexV1, +) -> Result<(), RbcDagError> { + bytes.push(CONSENSUS_ROUND_FIELD); + bytes.extend_from_slice(&vertex.consensus_round.to_be_bytes()); + bytes.push(STRONG_PARENTS_FIELD); + encode_count(bytes, "strong parents", vertex.strong_parents.len())?; + for parent in &vertex.strong_parents { + encode_consensus_reference(bytes, *parent); + } + bytes.push(DELIVERY_FRONTIER_FIELD); + encode_count(bytes, "delivery frontier", vertex.delivery_frontier.len())?; + for entry in &vertex.delivery_frontier { + match entry { + None => bytes.push(OPTION_NONE), + Some(reference) => { + bytes.push(OPTION_SOME); + encode_reference(bytes, *reference); + } + } + } + bytes.push(LEADER_CHOICE_FIELD); + match vertex.leader_choice { + LeaderChoiceV1::Vote { leader } => { + bytes.push(LEADER_VOTE); + encode_consensus_reference(bytes, leader); + } + LeaderChoiceV1::NoVote { + leader_author, + leader_round, + } => { + bytes.push(LEADER_NO_VOTE); + bytes.extend_from_slice(&leader_author.to_be_bytes()); + bytes.extend_from_slice(&leader_round.to_be_bytes()); + } + } + Ok(()) +} + +fn encode_count(bytes: &mut Vec, field: &'static str, count: usize) -> Result<(), RbcDagError> { + let count = u16::try_from(count).map_err(|_| RbcDagError::VectorTooLong { field, count })?; + bytes.extend_from_slice(&count.to_be_bytes()); + Ok(()) +} + +fn encode_reference(bytes: &mut Vec, reference: BlockReference) { + bytes.extend_from_slice(&reference.authority.to_be_bytes()); + bytes.extend_from_slice(&reference.round.to_be_bytes()); + bytes.extend_from_slice(reference.digest.as_ref()); +} + +fn encode_consensus_reference(bytes: &mut Vec, reference: ConsensusVertexReference) { + encode_reference(bytes, reference.carrier); + bytes.extend_from_slice(&reference.consensus_round.to_be_bytes()); +} + +fn physical_parents(header: &CarrierHeaderV1) -> Vec { + let mut parents = Vec::with_capacity(header.weak_parents.len() + 1); + parents.push(header.own_prev); + parents.extend_from_slice(&header.weak_parents); + parents +} + +fn compressed_acknowledgments( + header: &CarrierHeaderV1, +) -> Result<(u16, Vec), RbcDagError> { + let parents = physical_parents(header); + let acknowledged: BTreeSet<_> = header.data_acknowledgments.iter().copied().collect(); + let mut intersection = parents.len(); + while intersection > 0 && acknowledged.contains(&parents[intersection - 1]) { + intersection -= 1; + } + let shared: BTreeSet<_> = parents[intersection..].iter().copied().collect(); + let extras = header + .data_acknowledgments + .iter() + .copied() + .filter(|reference| !shared.contains(reference)) + .collect(); + let intersection = u16::try_from(intersection).map_err(|_| RbcDagError::VectorTooLong { + field: "physical parents", + count: parents.len(), + })?; + Ok((intersection, extras)) +} + +fn normalize_acknowledgments(header: &mut CarrierHeaderV1) -> Result<(), RbcDagError> { + let mut seen = BTreeSet::new(); + for acknowledgment in &header.data_acknowledgments { + if !seen.insert(*acknowledgment) { + return Err(RbcDagError::DuplicateAcknowledgment(*acknowledgment)); + } + } + let parents = physical_parents(header); + let (intersection, extras) = compressed_acknowledgments(header)?; + let mut normalized = parents[intersection as usize..].to_vec(); + normalized.extend(extras); + header.data_acknowledgments = normalized; + Ok(()) +} + +fn decode_header( + bytes: &[u8], + acknowledgment_encoding: AckEncoding, +) -> Result { + if bytes.len() > MAX_CARRIER_CONTENT_SIZE_V1 { + return Err(RbcDagError::ContentTooLarge(bytes.len())); + } + let mut decoder = Decoder::new(bytes); + decoder.expect_marker(CONTENT_FORMAT_FIELD)?; + let version = decoder.read_u8()?; + let expected_version = match acknowledgment_encoding { + AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, + AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, + }; + if version != expected_version { + return Err(RbcDagError::UnsupportedVersion(version)); + } + decoder.expect_marker(AUTHOR_FIELD)?; + let author = decoder.read_u16()?; + decoder.expect_marker(CARRIER_ROUND_FIELD)?; + let carrier_round = decoder.read_u32()?; + decoder.expect_marker(OWN_PREV_FIELD)?; + let own_prev = decoder.read_reference()?; + decoder.expect_marker(WEAK_PARENTS_FIELD)?; + let weak_count = decoder.read_count("weak parents", MAX_COMMITTEE_SIZE as usize - 1)?; + let weak_parents = decoder.read_references(weak_count)?; + decoder.expect_marker(TRANSACTIONS_COMMITMENT_FIELD)?; + let transactions_commitment = TransactionsCommitment::from_bytes(decoder.read_array()?); + decoder.expect_marker(ACKNOWLEDGMENTS_FIELD)?; + let data_acknowledgments = match acknowledgment_encoding { + AckEncoding::Expanded => { + let count = decoder.read_count("acknowledgments", u16::MAX as usize)?; + decoder.read_references(count)? + } + AckEncoding::Compressed => { + let intersection = decoder.read_u16()? as usize; + let extra_count = decoder.read_count("extra acknowledgments", u16::MAX as usize)?; + let extras = decoder.read_references(extra_count)?; + let mut parents = Vec::with_capacity(weak_parents.len() + 1); + parents.push(own_prev); + parents.extend_from_slice(&weak_parents); + if intersection > parents.len() { + return Err(RbcDagError::NonCanonicalAcknowledgments); + } + let mut acknowledgments = parents[intersection..].to_vec(); + acknowledgments.extend_from_slice(&extras); + let provisional = CarrierHeaderV1 { + author, + carrier_round, + own_prev, + weak_parents: weak_parents.clone(), + transactions_commitment, + data_acknowledgments: acknowledgments.clone(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: 0, + }; + let (canonical_intersection, canonical_extras) = + compressed_acknowledgments(&provisional)?; + if canonical_intersection as usize != intersection || canonical_extras != extras { + return Err(RbcDagError::NonCanonicalAcknowledgments); + } + acknowledgments + } + }; + decoder.expect_marker(PHASE_BATCH_FIELD)?; + let phase_count = decoder.read_count("phase statements", MAX_PHASE_STATEMENTS_V1)?; + let mut phase_batch = Vec::with_capacity(phase_count); + for _ in 0..phase_count { + let code = decoder.read_u8()?; + let target = decoder.read_reference()?; + phase_batch.push(match code { + PHASE_ECHO => RbcPhaseStatementV1::Echo { target }, + PHASE_READY => RbcPhaseStatementV1::Ready { target }, + other => return Err(RbcDagError::InvalidPhase(other)), + }); + } + decoder.expect_marker(CONSENSUS_VERTEX_FIELD)?; + let consensus_vertex = match decoder.read_u8()? { + OPTION_NONE => None, + OPTION_SOME => Some(decoder.read_consensus_vertex()?), + other => return Err(RbcDagError::InvalidOption(other)), + }; + decoder.expect_marker(CREATION_TIME_FIELD)?; + let creation_time_ns = decoder.read_u64()?; + decoder.finish()?; + Ok(CarrierHeaderV1 { + author, + carrier_round, + own_prev, + weak_parents, + transactions_commitment, + data_acknowledgments, + phase_batch, + consensus_vertex, + creation_time_ns, + }) +} + +struct Decoder<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> Decoder<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + fn take(&mut self, length: usize) -> Result<&'a [u8], RbcDagError> { + let end = self + .position + .checked_add(length) + .ok_or(RbcDagError::UnexpectedEnd)?; + let value = self + .bytes + .get(self.position..end) + .ok_or(RbcDagError::UnexpectedEnd)?; + self.position = end; + Ok(value) + } + + fn read_array(&mut self) -> Result<[u8; N], RbcDagError> { + let mut value = [0; N]; + value.copy_from_slice(self.take(N)?); + Ok(value) + } + + fn read_u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn read_u16(&mut self) -> Result { + Ok(u16::from_be_bytes(self.read_array()?)) + } + + fn read_u32(&mut self) -> Result { + Ok(u32::from_be_bytes(self.read_array()?)) + } + + fn read_u64(&mut self) -> Result { + Ok(u64::from_be_bytes(self.read_array()?)) + } + + fn expect_marker(&mut self, expected: u8) -> Result<(), RbcDagError> { + let actual = self.read_u8()?; + if actual != expected { + return Err(RbcDagError::InvalidMarker { expected, actual }); + } + Ok(()) + } + + fn read_count(&mut self, field: &'static str, maximum: usize) -> Result { + let count = self.read_u16()? as usize; + if count > maximum { + return Err(RbcDagError::VectorTooLong { field, count }); + } + Ok(count) + } + + fn read_reference(&mut self) -> Result { + let authority = self.read_u16()?; + let round = self.read_u32()?; + let digest = BlockDigest::from(self.read_array()?); + Ok(BlockReference { + round, + authority, + digest, + }) + } + + fn read_references(&mut self, count: usize) -> Result, RbcDagError> { + let byte_count = count + .checked_mul(BLOCK_REFERENCE_SIZE) + .ok_or(RbcDagError::UnexpectedEnd)?; + if self.bytes.len().saturating_sub(self.position) < byte_count { + return Err(RbcDagError::UnexpectedEnd); + } + let mut references = Vec::with_capacity(count); + for _ in 0..count { + references.push(self.read_reference()?); + } + Ok(references) + } + + fn read_consensus_reference(&mut self) -> Result { + Ok(ConsensusVertexReference::new( + self.read_reference()?, + self.read_u32()?, + )) + } + + fn read_consensus_vertex(&mut self) -> Result { + self.expect_marker(CONSENSUS_ROUND_FIELD)?; + let consensus_round = self.read_u32()?; + self.expect_marker(STRONG_PARENTS_FIELD)?; + let parent_count = self.read_count("strong parents", MAX_COMMITTEE_SIZE as usize)?; + let mut strong_parents = Vec::with_capacity(parent_count); + for _ in 0..parent_count { + strong_parents.push(self.read_consensus_reference()?); + } + self.expect_marker(DELIVERY_FRONTIER_FIELD)?; + let frontier_count = self.read_count("delivery frontier", MAX_COMMITTEE_SIZE as usize)?; + let mut delivery_frontier = Vec::with_capacity(frontier_count); + for _ in 0..frontier_count { + delivery_frontier.push(match self.read_u8()? { + OPTION_NONE => None, + OPTION_SOME => Some(self.read_reference()?), + other => return Err(RbcDagError::InvalidOption(other)), + }); + } + self.expect_marker(LEADER_CHOICE_FIELD)?; + let leader_choice = match self.read_u8()? { + LEADER_NONE => return Err(RbcDagError::InvalidLeaderChoice(LEADER_NONE)), + LEADER_VOTE => LeaderChoiceV1::Vote { + leader: self.read_consensus_reference()?, + }, + LEADER_NO_VOTE => LeaderChoiceV1::NoVote { + leader_author: self.read_u16()?, + leader_round: self.read_u32()?, + }, + other => return Err(RbcDagError::InvalidLeaderChoice(other)), + }; + Ok(ConsensusVertexV1::new( + consensus_round, + strong_parents, + delivery_frontier, + leader_choice, + )) + } + + fn finish(self) -> Result<(), RbcDagError> { + if self.position != self.bytes.len() { + return Err(RbcDagError::TrailingBytes(self.bytes.len() - self.position)); + } + Ok(()) + } +} + +fn validate_committee(committee: &Committee) -> Result<(), RbcDagError> { + if committee.is_empty() || committee.len() > MAX_COMMITTEE_SIZE as usize { + return Err(RbcDagError::InvalidCommittee("invalid committee size")); + } + let mut total_stake = 0u64; + for authority in committee.authorities() { + let stake = committee + .get_stake(authority) + .ok_or(RbcDagError::UnknownAuthority(authority))?; + if stake == 0 { + return Err(RbcDagError::InvalidCommittee("zero stake")); + } + total_stake = total_stake + .checked_add(stake) + .ok_or(RbcDagError::InvalidCommittee("stake overflow"))?; + } + let expected_validity = total_stake / 3 + 1; + let expected_quorum = total_stake + .checked_mul(2) + .ok_or(RbcDagError::InvalidCommittee("stake overflow"))? + / 3 + + 1; + if committee.validity_threshold() != expected_validity + || committee.quorum_threshold() != expected_quorum + { + return Err(RbcDagError::InvalidCommittee("threshold mismatch")); + } + let committee_size = committee.len(); + let fault_count = (committee_size - 1) / 3; + let expected_info_length = match committee_size % 3 { + 0 => fault_count + 3, + 1 => fault_count + 1, + _ => fault_count + 2, + }; + if committee.info_length() != expected_info_length { + return Err(RbcDagError::InvalidCommittee("information length mismatch")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::crypto::{ + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + }; + + fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { + BlockReference { + authority, + round, + digest: BlockDigest::from([marker; 32]), + } + } + + fn quorum_others(committee: &Committee, author: AuthorityIndex) -> Vec { + let mut stake = committee.get_stake(author).unwrap(); + let mut others = Vec::new(); + for authority in committee.authorities().filter(|other| *other != author) { + if stake >= committee.quorum_threshold() { + break; + } + stake += committee.get_stake(authority).unwrap(); + others.push(authority); + } + others.sort_unstable(); + others + } + + fn args( + committee: &Committee, + author: AuthorityIndex, + carrier_round: RoundNumber, + ) -> CarrierHeaderV1Args { + let previous = carrier_round - 1; + let parent = |authority| { + if previous == 0 { + carrier_genesis_reference(authority) + } else { + reference(authority, previous, 0x40 + authority as u8) + } + }; + CarrierHeaderV1Args { + author, + carrier_round, + own_prev: parent(author), + weak_parents: quorum_others(committee, author) + .into_iter() + .map(parent) + .collect(), + transactions_commitment: TransactionsCommitment::from_bytes([0x55; 32]), + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: 0x0102_0304_0506_0708, + } + } + + fn full_args(committee: &Committee) -> CarrierHeaderV1Args { + let mut args = args(committee, 3, 2); + let phase_target = reference(2, 1, 0x72); + args.data_acknowledgments = vec![*args.weak_parents.last().unwrap(), reference(2, 1, 0x61)]; + args.phase_batch = vec![ + RbcPhaseStatementV1::Echo { + target: phase_target, + }, + RbcPhaseStatementV1::Ready { + target: phase_target, + }, + ]; + let strong_parents = [0, 1, 3] + .into_iter() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + args.consensus_vertex = Some(ConsensusVertexV1::new( + 1, + strong_parents, + vec![None; committee.len()], + LeaderChoiceV1::Vote { + leader: ConsensusVertexReference::new(carrier_genesis_reference(0), 0), + }, + )); + args + } + + fn full_candidate(committee: &Committee) -> CandidateCarrierV1 { + CandidateCarrierV1::try_new(full_args(committee), committee).unwrap() + } + + #[test] + fn canonical_content_and_reference_have_frozen_golden_bytes() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + let bytes = candidate.canonical_content_bytes().unwrap(); + assert_eq!( + bytes[0..2], + [CONTENT_FORMAT_FIELD, CARRIER_FORMAT_VERSION_V1] + ); + assert_eq!( + candidate.reference().digest.as_ref(), + blake3::hash(&bytes).as_bytes() + ); + let mut expected = vec![0x00, 0x01, 0x01]; + expected.extend_from_slice(&3u16.to_be_bytes()); + expected.push(0x02); + expected.extend_from_slice(&2u32.to_be_bytes()); + expected.push(0x03); + encode_reference(&mut expected, reference(3, 1, 0x43)); + expected.extend_from_slice(&[0x04, 0x00, 0x02]); + encode_reference(&mut expected, reference(0, 1, 0x40)); + encode_reference(&mut expected, reference(1, 1, 0x41)); + expected.push(0x05); + expected.extend_from_slice(&[0x55; 32]); + expected.extend_from_slice(&[0x06, 0x00, 0x02]); + encode_reference(&mut expected, reference(1, 1, 0x41)); + encode_reference(&mut expected, reference(2, 1, 0x61)); + expected.extend_from_slice(&[0x07, 0x00, 0x02]); + for phase in [0x00, 0x01] { + expected.push(phase); + encode_reference(&mut expected, reference(2, 1, 0x72)); + } + expected.extend_from_slice(&[0x08, 0x01, 0x01]); + expected.extend_from_slice(&1u32.to_be_bytes()); + expected.extend_from_slice(&[0x02, 0x00, 0x03]); + for authority in [0, 1, 3] { + encode_reference(&mut expected, carrier_genesis_reference(authority)); + expected.extend_from_slice(&0u32.to_be_bytes()); + } + expected.extend_from_slice(&[0x03, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00]); + expected.extend_from_slice(&[0x04, 0x01]); + encode_reference(&mut expected, carrier_genesis_reference(0)); + expected.extend_from_slice(&0u32.to_be_bytes()); + expected.push(0x09); + expected.extend_from_slice(&0x0102_0304_0506_0708u64.to_be_bytes()); + assert_eq!(bytes, expected); + assert_eq!( + hex::encode(candidate.reference().digest.as_ref()), + "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f" + ); + } + + #[test] + fn every_canonical_carrier_field_is_bound_to_the_reference() { + let committee = Committee::new_test(vec![1; 4]); + let mut base = full_args(&committee); + base.data_acknowledgments.push(reference(0, 1, 0x62)); + let base_candidate = CandidateCarrierV1::try_new(base.clone(), &committee).unwrap(); + let base_reference = base_candidate.reference(); + let mut mutations = Vec::new(); + + let mut changed = base.clone(); + changed.author = 2; + changed.own_prev = reference(2, 1, 0x42); + mutations.push(("author", changed)); + + let mut changed = base.clone(); + changed.carrier_round = 3; + changed.own_prev = reference(3, 2, 0x43); + changed.weak_parents = vec![reference(0, 2, 0x40), reference(1, 2, 0x41)]; + mutations.push(("carrier round", changed)); + + let mut changed = base.clone(); + changed.own_prev.digest = BlockDigest::from([0x91; 32]); + mutations.push(("own predecessor", changed)); + + let mut changed = base.clone(); + changed.weak_parents[0].digest = BlockDigest::from([0x92; 32]); + mutations.push(("weak parent", changed)); + + let mut changed = base.clone(); + changed.weak_parents.push(reference(2, 1, 0x42)); + mutations.push(("weak parent count", changed)); + + let mut changed = base.clone(); + changed.transactions_commitment = TransactionsCommitment::from_bytes([0x93; 32]); + mutations.push(("transaction commitment", changed)); + + let mut changed = base.clone(); + changed.data_acknowledgments[1].digest = BlockDigest::from([0x94; 32]); + mutations.push(("acknowledgment", changed)); + + let mut changed = base.clone(); + changed.data_acknowledgments.push(reference(3, 1, 0x95)); + mutations.push(("acknowledgment count", changed)); + + let mut changed = base.clone(); + changed.data_acknowledgments.swap(1, 2); + mutations.push(("acknowledgment order", changed)); + + let mut changed = base.clone(); + changed.phase_batch[0] = RbcPhaseStatementV1::Ready { + target: reference(1, 1, 0x96), + }; + mutations.push(("phase kind", changed)); + + let mut changed = base.clone(); + changed.phase_batch[0] = RbcPhaseStatementV1::Echo { + target: reference(2, 1, 0x97), + }; + mutations.push(("phase target", changed)); + + let mut changed = base.clone(); + changed.phase_batch.swap(0, 1); + mutations.push(("phase order", changed)); + + let mut changed = base.clone(); + changed.phase_batch.pop(); + mutations.push(("phase count", changed)); + + let mut changed = base.clone(); + changed.consensus_vertex = None; + mutations.push(("consensus presence", changed)); + + let mut changed = base.clone(); + changed.consensus_vertex.as_mut().unwrap().consensus_round = 2; + mutations.push(("consensus round", changed)); + + let mut changed = base.clone(); + changed.consensus_vertex.as_mut().unwrap().strong_parents[0] + .carrier + .digest = BlockDigest::from([0x98; 32]); + mutations.push(("strong parent carrier", changed)); + + let mut changed = base.clone(); + changed.consensus_vertex.as_mut().unwrap().strong_parents[0].consensus_round = 1; + mutations.push(("strong parent consensus round", changed)); + + let mut changed = base.clone(); + changed + .consensus_vertex + .as_mut() + .unwrap() + .strong_parents + .insert( + 2, + ConsensusVertexReference::new(carrier_genesis_reference(2), 0), + ); + mutations.push(("strong parent count", changed)); + + let mut changed = base.clone(); + changed.consensus_vertex.as_mut().unwrap().delivery_frontier[0] = + Some(reference(0, 1, 0x99)); + mutations.push(("frontier entry", changed)); + + let mut changed = base.clone(); + changed + .consensus_vertex + .as_mut() + .unwrap() + .delivery_frontier + .pop(); + mutations.push(("frontier count", changed)); + + let mut changed = base.clone(); + changed.consensus_vertex.as_mut().unwrap().leader_choice = LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }; + mutations.push(("leader choice", changed)); + + let mut changed = base; + changed.creation_time_ns ^= 1; + mutations.push(("creation time", changed)); + + for (field, changed) in mutations { + let candidate = CandidateCarrierV1::try_new(changed, &committee) + .unwrap_or_else(|error| panic!("{field} mutation must remain encodable: {error}")); + assert_ne!(candidate.reference(), base_reference, "unbound {field}"); + let bytes = candidate.canonical_content_bytes().unwrap(); + assert!(matches!( + CandidateCarrierV1::decode_content(&bytes, &committee, Some(base_reference)), + Err(RbcDagError::ReferenceMismatch { .. }) + )); + } + } + + #[test] + fn content_and_compressed_wire_round_trip_to_same_reference() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + let content = candidate.canonical_content_bytes().unwrap(); + let wire = candidate.canonical_wire_bytes().unwrap(); + assert_eq!(content[1], CARRIER_FORMAT_VERSION_V1); + assert_eq!(wire[1], CARRIER_WIRE_FORMAT_VERSION_V1); + assert!(wire.len() < content.len()); + assert_eq!( + CandidateCarrierV1::decode_content(&content, &committee, Some(candidate.reference())) + .unwrap(), + candidate + ); + assert!(matches!( + CandidateCarrierV1::decode_content(&wire, &committee, None), + Err(RbcDagError::UnsupportedVersion( + CARRIER_WIRE_FORMAT_VERSION_V1 + )) + )); + assert!(matches!( + CandidateCarrierV1::decode_wire(&content, &committee, None), + Err(RbcDagError::UnsupportedVersion(CARRIER_FORMAT_VERSION_V1)) + )); + assert_eq!( + CandidateCarrierV1::decode_wire(&wire, &committee, Some(candidate.reference())) + .unwrap(), + candidate + ); + for end in 0..content.len() { + assert!(CandidateCarrierV1::decode_content(&content[..end], &committee, None).is_err()); + } + for end in 0..wire.len() { + assert!(CandidateCarrierV1::decode_wire(&wire[..end], &committee, None).is_err()); + } + let mut trailing = wire; + trailing.push(0); + assert!(matches!( + CandidateCarrierV1::decode_wire(&trailing, &committee, None), + Err(RbcDagError::TrailingBytes(1)) + )); + } + + #[test] + fn acknowledgment_compression_normalizes_and_rejects_duplicates() { + let committee = Committee::new_test(vec![1; 4]); + let base = args(&committee, 3, 2); + let shared = *base.weak_parents.last().unwrap(); + let extra = reference(2, 1, 0x91); + + let mut reordered = base.clone(); + reordered.data_acknowledgments = vec![extra, shared]; + let candidate = CandidateCarrierV1::try_new(reordered, &committee).unwrap(); + assert_eq!(candidate.header().data_acknowledgments(), &[shared, extra]); + + for acknowledgments in [vec![shared, shared], vec![extra, shared, shared]] { + let mut duplicate = base.clone(); + duplicate.data_acknowledgments = acknowledgments; + assert!(matches!( + CandidateCarrierV1::try_new(duplicate, &committee), + Err(RbcDagError::DuplicateAcknowledgment(reference)) if reference == shared + )); + } + } + + #[test] + fn acknowledgment_validation_handles_many_shared_hash_prefixes() { + let committee = Committee::new_test(vec![1; 4]); + let mut carrier = args(&committee, 3, 2); + carrier.data_acknowledgments = (0..8_192u64) + .map(|counter| { + let mut digest = [0xA5; 32]; + digest[24..].copy_from_slice(&counter.to_be_bytes()); + BlockReference { + authority: 2, + round: 1, + digest: BlockDigest::from(digest), + } + }) + .collect(); + + let candidate = CandidateCarrierV1::try_new(carrier, &committee).unwrap(); + assert_eq!(candidate.header().data_acknowledgments().len(), 8_192); + assert!(candidate.canonical_content_bytes().unwrap().len() < MAX_CARRIER_CONTENT_SIZE_V1); + } + + #[test] + fn phase_slot_conflict_is_rejected_even_when_digests_differ() { + let committee = Committee::new_test(vec![1; 4]); + let mut args = args(&committee, 3, 2); + args.phase_batch = vec![ + RbcPhaseStatementV1::Echo { + target: reference(0, 1, 0x01), + }, + RbcPhaseStatementV1::Echo { + target: reference(0, 1, 0x02), + }, + ]; + assert!(matches!( + CandidateCarrierV1::try_new(args, &committee), + Err(RbcDagError::DuplicatePhaseStatement(_)) + )); + } + + #[test] + fn malformed_optional_vertex_does_not_invalidate_outer_candidate() { + let committee = Committee::new_test(vec![1; 4]); + let mut args = args(&committee, 3, 1); + args.consensus_vertex = Some(ConsensusVertexV1::new( + 1, + Vec::new(), + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }, + )); + let candidate = CandidateCarrierV1::try_new(args, &committee).unwrap(); + assert!(matches!( + candidate.validate_consensus_vertex(&committee), + Err(RbcDagProjectionError::InvalidStrongParentThreshold) + )); + } + + #[test] + fn valid_projection_shape_checks_quorum_frontier_and_choice() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + assert!( + candidate + .validate_consensus_vertex(&committee) + .unwrap() + .is_some() + ); + } + + #[test] + fn auth_statement_has_frozen_layout() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + let instance = RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(); + let context = + RbcDagContextV1::new(instance, &committee, BlockAuthenticationScheme::MlDsa65).unwrap(); + let base = context.public_authentication_statement(candidate.reference()); + assert_eq!( + hex::encode(context.committee_id().as_bytes()), + "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62" + ); + assert_eq!( + hex::encode(base), + concat!( + "53544152464953485f5242435f4441475f56310002", + "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", + "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", + "000300000002", + "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f" + ) + ); + assert_eq!( + hex::encode(context.public_authentication_digest(candidate.reference())), + "26a0866c9c6938c9158495f23fd22b281db6371461b6aad109ad41329e5fa5c8" + ); + assert_eq!(&base[..19], AUTHENTICATION_DOMAIN); + assert_eq!(base[19], CARRIER_AUTHENTICATION_KIND); + assert_eq!(base[20], 2); + assert_eq!(&base[21..53], &[0xA5; 32]); + assert_eq!(&base[53..85], context.committee_id().as_bytes()); + assert_eq!(&base[85..87], &3u16.to_be_bytes()); + assert_eq!(&base[87..91], &2u32.to_be_bytes()); + assert_eq!(&base[91..], candidate.reference().digest.as_ref()); + let public_digest = blake3::hash(&base); + for (field, offset) in [ + ("domain", 0), + ("kind", 19), + ("scheme", 20), + ("instance", 21), + ("committee", 53), + ("author", 85), + ("round", 87), + ("content digest", 91), + ] { + let mut changed = base; + changed[offset] ^= 1; + assert_ne!( + blake3::hash(&changed), + public_digest, + "{field} must be bound by the public authenticator" + ); + } + + let mac_context = + RbcDagContextV1::new(instance, &committee, BlockAuthenticationScheme::MacVector) + .unwrap(); + let keyrings = mac_keyrings_for_test(committee.len()); + let mac_statement = mac_context.mac_authentication_statement(candidate.reference(), 2); + assert_eq!( + hex::encode(mac_statement), + concat!( + "53544152464953485f5242435f4441475f56310003", + "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", + "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", + "000300000002", + "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f", + "0002" + ) + ); + let key = &keyrings[3][2]; + let tag = key.compute_rbc_tag(&mac_statement); + assert_eq!( + hex::encode(tag.as_ref()), + "118209f3c2c3025918ae7f60fe5a04a94e639cd06d910d89c483035c014fff02" + ); + for (field, offset) in [ + ("domain", 0), + ("kind", 19), + ("scheme", 20), + ("instance", 21), + ("committee", 53), + ("author", 85), + ("round", 87), + ("content digest", 91), + ("recipient", 123), + ] { + let mut changed = mac_statement; + changed[offset] ^= 1; + assert_ne!( + key.compute_rbc_tag(&changed), + tag, + "{field} must be bound by the MAC" + ); + } + } + + #[test] + fn authorizer_is_bound_to_the_claimed_author_and_committee_key() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + let context = RbcDagContextV1::new( + RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::Ed25519, + ) + .unwrap(); + let committee_signer = dummy_signer(); + + assert!(matches!( + context.authenticate( + &candidate, + &committee, + CarrierAuthorizerV1::Ed25519 { + authority: 2, + signer: &committee_signer, + }, + ), + Err(RbcDagError::AuthorizerAuthorityMismatch { + expected: 3, + actual: 2, + }) + )); + + let other_signers = Signer::new_for_test(1); + assert!(matches!( + context.authenticate( + &candidate, + &committee, + CarrierAuthorizerV1::Ed25519 { + authority: 3, + signer: &other_signers[0], + }, + ), + Err(RbcDagError::AuthorizerKeyMismatch) + )); + } + + #[test] + fn every_authentication_scheme_round_trips_and_keeps_content_identity() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + let instance = RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(); + let keyrings = mac_keyrings_for_test(committee.len()); + let ed_signer = dummy_signer(); + let ml44_signer = dummy_ml_dsa_44_signer(); + let ml65_signer = dummy_ml_dsa_65_signer(); + + for (scheme, authorizer) in [ + ( + BlockAuthenticationScheme::Ed25519, + CarrierAuthorizerV1::Ed25519 { + authority: 3, + signer: &ed_signer, + }, + ), + ( + BlockAuthenticationScheme::MlDsa44, + CarrierAuthorizerV1::MlDsa44 { + authority: 3, + signer: &ml44_signer, + }, + ), + ( + BlockAuthenticationScheme::MlDsa65, + CarrierAuthorizerV1::MlDsa65 { + authority: 3, + signer: &ml65_signer, + }, + ), + ( + BlockAuthenticationScheme::MacVector, + CarrierAuthorizerV1::MacVector { + authority: 3, + keys: &keyrings[3], + }, + ), + ] { + let context = RbcDagContextV1::new(instance, &committee, scheme).unwrap(); + let authentication = context + .authenticate(&candidate, &committee, authorizer) + .unwrap(); + let wire = authentication.canonical_wire_bytes(); + let payload_len = match scheme { + BlockAuthenticationScheme::Ed25519 => SIGNATURE_SIZE, + BlockAuthenticationScheme::MlDsa44 => ML_DSA_44_SIGNATURE_SIZE, + BlockAuthenticationScheme::MlDsa65 => ML_DSA_65_SIGNATURE_SIZE, + BlockAuthenticationScheme::MacVector => committee.len() * MAC_TAG_SIZE, + }; + assert_eq!( + &wire[..3], + &[ + CONTENT_FORMAT_FIELD, + CARRIER_FORMAT_VERSION_V1, + authentication_scheme_code(scheme), + ] + ); + assert_eq!(wire.len(), 3 + payload_len); + let decoded = CarrierAuthenticationV1::decode_wire(&wire, &committee).unwrap(); + assert_eq!(decoded, authentication); + let authenticated = context + .verify_authentication(candidate.clone(), decoded, 1, &committee, &keyrings[1]) + .unwrap(); + assert_eq!(authenticated.reference(), candidate.reference()); + assert_eq!(authenticated.receiver(), 1); + assert_eq!(authenticated.context(), context); + } + } + + #[test] + fn mac_verifies_only_local_entry_and_context() { + let committee = Committee::new_test(vec![1; 4]); + let candidate = full_candidate(&committee); + let keyrings = mac_keyrings_for_test(committee.len()); + let instance = RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(); + let context = + RbcDagContextV1::new(instance, &committee, BlockAuthenticationScheme::MacVector) + .unwrap(); + let authentication = context + .authenticate( + &candidate, + &committee, + CarrierAuthorizerV1::MacVector { + authority: 3, + keys: &keyrings[3], + }, + ) + .unwrap(); + let CarrierAuthenticationV1::MacVector(vector) = authentication else { + unreachable!() + }; + let mut poisoned_other = vector.as_bytes().to_vec(); + poisoned_other[2 * MAC_TAG_SIZE] ^= 0xFF; + let poisoned_other = + CarrierAuthenticationV1::MacVector(FlatMacVector::from_bytes(poisoned_other).unwrap()); + assert!( + context + .verify_authentication( + candidate.clone(), + poisoned_other.clone(), + 1, + &committee, + &keyrings[1], + ) + .is_ok() + ); + assert!(matches!( + context.verify_authentication( + candidate.clone(), + poisoned_other, + 2, + &committee, + &keyrings[2], + ), + Err(RbcDagError::InvalidAuthentication) + )); + + let other_context = RbcDagContextV1::new( + RbcDagProtocolInstanceId::new([0xB6; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ) + .unwrap(); + let original = context + .authenticate( + &candidate, + &committee, + CarrierAuthorizerV1::MacVector { + authority: 3, + keys: &keyrings[3], + }, + ) + .unwrap(); + assert!(matches!( + other_context.verify_authentication(candidate, original, 1, &committee, &keyrings[1],), + Err(RbcDagError::InvalidAuthentication) + )); + } + + #[test] + fn candidate_and_capability_are_committee_bound() { + let committee = Committee::new_test(vec![1; 4]); + let other_committee = Committee::new_test(vec![1, 1, 1, 2]); + let candidate = full_candidate(&committee); + let context = RbcDagContextV1::new( + RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(), + &other_committee, + BlockAuthenticationScheme::Ed25519, + ) + .unwrap(); + assert!(matches!( + context.authenticate( + &candidate, + &other_committee, + CarrierAuthorizerV1::Ed25519 { + authority: 3, + signer: &dummy_signer(), + }, + ), + Err(RbcDagError::CandidateCommitteeMismatch) + )); + assert!(matches!( + candidate.validate_consensus_vertex(&other_committee), + Err(RbcDagProjectionError::CommitteeMismatch) + )); + } + + #[test] + fn sidecar_wire_has_frozen_flat_mac_shape() { + let committee = Committee::new_test(vec![1; 4]); + let tags = (0..4) + .map(|index| MacTag::from_bytes([index; MAC_TAG_SIZE])) + .collect::>(); + let authentication = + CarrierAuthenticationV1::MacVector(FlatMacVector::from_tags(&tags).unwrap()); + let wire = authentication.canonical_wire_bytes(); + assert_eq!(&wire[..3], &[0x00, 0x01, 0x03]); + assert_eq!(wire.len(), 3 + committee.len() * MAC_TAG_SIZE); + assert_eq!( + hex::encode(wire), + concat!( + "000103", + "0000000000000000000000000000000000000000000000000000000000000000", + "0101010101010101010101010101010101010101010101010101010101010101", + "0202020202020202020202020202020202020202020202020202020202020202", + "0303030303030303030303030303030303030303030303030303030303030303" + ) + ); + + let mut trailing = authentication.canonical_wire_bytes(); + trailing.push(0xFF); + assert!(matches!( + CarrierAuthenticationV1::decode_wire(&trailing, &committee), + Err(RbcDagError::TrailingBytes(1)) + )); + let mut truncated = authentication.canonical_wire_bytes(); + truncated.pop(); + assert!(matches!( + CarrierAuthenticationV1::decode_wire(&truncated, &committee), + Err(RbcDagError::UnexpectedEnd) + )); + } +} diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs new file mode 100644 index 00000000..f1506bad --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -0,0 +1,2266 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Deterministic, in-memory model for the embedded-RBC carrier DAG. +//! +//! This reducer intentionally has no networking, storage, timers, or consensus +//! integration. Authentication is represented by a capability supplied by the +//! caller after the outer authenticator has been checked. This keeps the +//! important authority boundary explicit: canonical but unauthenticated +//! content may help header recovery, while only the first authenticated value +//! in a carrier slot is optimistically admitted and allowed to ECHO. + +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + error::Error, + fmt, + sync::Arc, +}; + +use crate::{ + committee::Committee, + types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, +}; + +use super::{ + AuthenticatedCarrierV1, CandidateCarrierV1, LocallyAuthenticatedCarrierV1, + MAX_PHASE_STATEMENTS_V1, RbcDagCommitteeId, RbcDagContextV1, RbcPhaseStatementV1, + carrier_genesis_reference, +}; + +/// Executable-model runahead bound. This is deliberately a model parameter, +/// not a production protocol constant; the runtime value remains a proof and +/// benchmarking decision. +pub const EXECUTABLE_MODEL_ADMISSION_WINDOW_V1: RoundNumber = 2; +pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 4; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum IngressAuthentication { + Authenticated, + CandidateOnly, +} + +/// Observable effects of one deterministic reducer transition. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ModelEffect { + /// A threshold is latched, but the exact canonical carrier is absent. + NeedCarrier { + target: BlockReference, + holders: Vec, + }, + /// The local Bracha instance delivered this exact carrier value. + Delivered(BlockReference), + /// One exact author prefix advanced by one carrier. + PrefixAdvanced { + authority: AuthorityIndex, + tip: BlockReference, + }, + /// The sequential fast clock opened the next local carrier round. + CarrierRoundAdvanced(RoundNumber), +} + +/// Snapshot of the lifecycle predicates for one exact carrier. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CarrierLifecycle { + pub authenticated: bool, + pub admitted: bool, + pub phase_batch_processed: bool, + pub delivered: bool, + pub data_available: bool, + pub prefix_closed: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ModelError { + InvalidCommittee, + CommitteeMismatch { + expected: RbcDagCommitteeId, + actual: RbcDagCommitteeId, + }, + ContextMismatch, + AuthenticationReceiverMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + UnknownAuthority(AuthorityIndex), + LocalAuthorMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + LocalCarrierRequiresStart(BlockReference), + LocalRoundNotOpen(RoundNumber), + UnexpectedLocalRound { + expected: RoundNumber, + actual: RoundNumber, + }, + LocalCarrierAlreadyFixed(RoundNumber), + FutureCarrierOutsideBuffer { + current: RoundNumber, + maximum: RoundNumber, + actual: RoundNumber, + }, + WrongLocalPredecessor { + expected: BlockReference, + actual: BlockReference, + }, + LocalWeakParentNotAdmitted(BlockReference), + LocalPhaseBatchMismatch, + ConflictingCarrierContent(BlockReference), + UnexpectedRecovery(BlockReference), + MissingCarrier(BlockReference), + FrontierLength { + expected: usize, + actual: usize, + }, + FrontierAuthority { + index: AuthorityIndex, + reference: BlockReference, + }, + FrontierNotClosed(BlockReference), + FrontierRegression { + authority: AuthorityIndex, + previous: Option, + proposed: Option, + }, +} + +impl fmt::Display for ModelError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "Starfish-RBC-DAG model error: {self:?}") + } +} + +impl Error for ModelError {} + +#[derive(Clone)] +struct CarrierRecord { + carrier: CandidateCarrierV1, + authenticated: bool, + admitted: bool, + phase_batch_cursor: usize, + delivered: bool, + data_available: bool, + prefix_closed: bool, +} + +impl CarrierRecord { + fn new(carrier: CandidateCarrierV1) -> Self { + Self { + carrier, + authenticated: false, + admitted: false, + phase_batch_cursor: 0, + delivered: false, + data_available: false, + prefix_closed: false, + } + } + + fn lifecycle(&self) -> CarrierLifecycle { + CarrierLifecycle { + authenticated: self.authenticated, + admitted: self.admitted, + phase_batch_processed: self.phase_batch_cursor + == self.carrier.header().phase_batch().len(), + delivered: self.delivered, + data_available: self.data_available, + prefix_closed: self.prefix_closed, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct RbcCandidateState { + echoes: BTreeSet, + readies: BTreeSet, + echo_quorum_observed: bool, + ready_validity_observed: bool, + ready_quorum_observed: bool, + requested_holders: BTreeSet, +} + +impl RbcCandidateState { + fn holders(&self) -> BTreeSet { + self.echoes.union(&self.readies).copied().collect() + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct RbcSlotState { + echoed: Option, + readied: Option, + delivered: Option, + echo_by_sender: BTreeMap, + ready_by_sender: BTreeMap, + candidates: BTreeMap, +} + +#[derive(Clone, Copy)] +enum RbcAction { + NeedCarrier, + SendReady, + Deliver, + None, +} + +/// Pure local state machine used by the milestone-two simulations. +#[derive(Clone)] +pub struct RbcDagModel { + committee: Arc, + committee_id: RbcDagCommitteeId, + context: RbcDagContextV1, + own_authority: AuthorityIndex, + local_carrier_round: RoundNumber, + own_fixed: BTreeMap, + carriers: BTreeMap, + authenticated_by_slot: BTreeMap<(RoundNumber, AuthorityIndex), BlockReference>, + admitted_by_slot: BTreeMap<(RoundNumber, AuthorityIndex), BlockReference>, + rbc_slots: BTreeMap<(RoundNumber, AuthorityIndex), RbcSlotState>, + pending_phases: VecDeque, + pending_phase_set: BTreeSet, + pending_delivered_batch_replays: VecDeque, + prefix_tips: Vec, + included_frontier: Vec>, + included: BTreeSet, +} + +impl RbcDagModel { + pub fn new( + committee: Arc, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + ) -> Result { + if !committee.known_authority(own_authority) { + return Err(ModelError::UnknownAuthority(own_authority)); + } + let committee_id = + RbcDagCommitteeId::derive(&committee).map_err(|_| ModelError::InvalidCommittee)?; + if context.committee_id() != committee_id { + return Err(ModelError::ContextMismatch); + } + let prefix_tips = committee + .authorities() + .map(carrier_genesis_reference) + .collect(); + Ok(Self { + included_frontier: vec![None; committee.len()], + committee, + committee_id, + context, + own_authority, + local_carrier_round: 1, + own_fixed: BTreeMap::new(), + carriers: BTreeMap::new(), + authenticated_by_slot: BTreeMap::new(), + admitted_by_slot: BTreeMap::new(), + rbc_slots: BTreeMap::new(), + pending_phases: VecDeque::new(), + pending_phase_set: BTreeSet::new(), + pending_delivered_batch_replays: VecDeque::new(), + prefix_tips, + included: BTreeSet::new(), + }) + } + + pub fn own_authority(&self) -> AuthorityIndex { + self.own_authority + } + + pub fn context(&self) -> RbcDagContextV1 { + self.context + } + + /// Current sequential local carrier slot. If `can_create_carrier` is + /// false, the local carrier is fixed and waits for exact-round quorum. + pub fn local_carrier_round(&self) -> RoundNumber { + self.local_carrier_round + } + + pub fn can_create_carrier(&self) -> bool { + !self.own_fixed.contains_key(&self.local_carrier_round) + } + + pub fn pending_phase_batch(&self) -> Vec { + let limit = self + .committee + .len() + .saturating_mul(4) + .min(MAX_PHASE_STATEMENTS_V1); + self.pending_phases + .iter() + .filter(|statement| statement.target().round < self.local_carrier_round) + .take(limit) + .copied() + .collect() + } + + pub fn pending_phase_backlog_len(&self) -> usize { + self.pending_phases.len() + } + + /// Return the exact predecessor and one deterministic quorum of admitted + /// weak parents for the next honest carrier. + pub fn local_parent_set(&self) -> Result<(BlockReference, Vec), ModelError> { + if !self.can_create_carrier() { + return Err(ModelError::LocalCarrierAlreadyFixed( + self.local_carrier_round, + )); + } + let parent_round = self.local_carrier_round - 1; + let own_prev = if parent_round == 0 { + carrier_genesis_reference(self.own_authority) + } else { + *self + .own_fixed + .get(&parent_round) + .ok_or(ModelError::LocalRoundNotOpen(self.local_carrier_round))? + }; + + let mut stake = self.authority_stake(self.own_authority); + let mut weak_parents = Vec::new(); + for authority in self.committee.authorities() { + if authority == self.own_authority { + continue; + } + let parent = if parent_round == 0 { + carrier_genesis_reference(authority) + } else { + let Some(parent) = self + .admitted_by_slot + .get(&(parent_round, authority)) + .copied() + else { + continue; + }; + parent + }; + weak_parents.push(parent); + stake = stake.saturating_add(self.authority_stake(authority)); + if stake >= self.committee.quorum_threshold() { + break; + } + } + if stake < self.committee.quorum_threshold() { + return Err(ModelError::LocalRoundNotOpen(self.local_carrier_round)); + } + Ok((own_prev, weak_parents)) + } + + /// Fix a locally authored carrier. Its phase batch must be the exact + /// bounded FIFO prefix returned by [`Self::pending_phase_batch`]; newly + /// generated ECHO is left for a later carrier. + pub fn start_local_carrier( + &mut self, + authenticated: LocallyAuthenticatedCarrierV1, + ) -> Result, ModelError> { + self.ensure_locally_authenticated(&authenticated)?; + let carrier = authenticated.candidate().clone(); + self.ensure_committee(&carrier)?; + let header = carrier.header(); + if header.author() != self.own_authority { + return Err(ModelError::LocalAuthorMismatch { + expected: self.own_authority, + actual: header.author(), + }); + } + if header.carrier_round() != self.local_carrier_round { + return Err(ModelError::UnexpectedLocalRound { + expected: self.local_carrier_round, + actual: header.carrier_round(), + }); + } + if !self.can_create_carrier() { + return Err(ModelError::LocalCarrierAlreadyFixed( + self.local_carrier_round, + )); + } + let (expected_prev, _) = self.local_parent_set()?; + if header.own_prev() != expected_prev { + return Err(ModelError::WrongLocalPredecessor { + expected: expected_prev, + actual: header.own_prev(), + }); + } + if header.carrier_round() > 1 { + for parent in header.weak_parents() { + if self.admitted_by_slot.get(&(parent.round, parent.authority)) != Some(parent) { + return Err(ModelError::LocalWeakParentNotAdmitted(*parent)); + } + } + } + let expected_phase_batch = self.pending_phase_batch(); + if header.phase_batch() != expected_phase_batch { + return Err(ModelError::LocalPhaseBatchMismatch); + } + + let round = header.carrier_round(); + let selected_phase_indices: BTreeSet<_> = self + .pending_phases + .iter() + .enumerate() + .filter(|(_, statement)| statement.target().round < round) + .take(expected_phase_batch.len()) + .map(|(index, _)| index) + .collect(); + let reference = carrier.reference(); + // Preflight all fallible ingress checks before the proof-critical + // write order. The exact local carrier must be fixed before its local + // ECHO is authorized or any embedded phase statement is exposed. + self.preflight_receive(&carrier)?; + self.own_fixed.insert(round, reference); + for statement in &expected_phase_batch { + self.pending_phase_set.remove(statement); + } + self.pending_phases = self + .pending_phases + .drain(..) + .enumerate() + .filter_map(|(index, statement)| { + (!selected_phase_indices.contains(&index)).then_some(statement) + }) + .collect(); + let mut effects = + self.apply_received_carrier(carrier, IngressAuthentication::Authenticated); + self.maybe_advance_fast_clock(&mut effects); + Ok(effects) + } + + /// Stage canonical content without granting optimistic admission or ECHO. + pub fn stage_candidate( + &mut self, + carrier: CandidateCarrierV1, + ) -> Result, ModelError> { + self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) + } + + /// Admit a carrier only through the opaque capability produced by the + /// codec's context- and receiver-bound authenticator verifier. + pub fn receive_authenticated( + &mut self, + authenticated: AuthenticatedCarrierV1, + ) -> Result, ModelError> { + self.ensure_authenticated(&authenticated)?; + if authenticated.candidate().header().author() == self.own_authority { + return Err(ModelError::LocalCarrierRequiresStart( + authenticated.candidate().reference(), + )); + } + self.receive_carrier( + authenticated.candidate().clone(), + IngressAuthentication::Authenticated, + ) + } + + fn receive_carrier( + &mut self, + carrier: CandidateCarrierV1, + authentication: IngressAuthentication, + ) -> Result, ModelError> { + self.preflight_receive(&carrier)?; + Ok(self.apply_received_carrier(carrier, authentication)) + } + + fn preflight_receive(&self, carrier: &CandidateCarrierV1) -> Result<(), ModelError> { + self.ensure_committee(carrier)?; + let reference = carrier.reference(); + let maximum = self + .local_carrier_round + .saturating_add(EXECUTABLE_MODEL_BUFFER_WINDOW_V1); + if reference.round > maximum { + return Err(ModelError::FutureCarrierOutsideBuffer { + current: self.local_carrier_round, + maximum, + actual: reference.round, + }); + } + let author = carrier.header().author(); + if !self.committee.known_authority(author) { + return Err(ModelError::UnknownAuthority(author)); + } + match self.carriers.get(&reference) { + Some(existing) if &existing.carrier != carrier => { + return Err(ModelError::ConflictingCarrierContent(reference)); + } + Some(_) | None => {} + } + Ok(()) + } + + fn apply_received_carrier( + &mut self, + carrier: CandidateCarrierV1, + authentication: IngressAuthentication, + ) -> Vec { + let reference = carrier.reference(); + self.carriers + .entry(reference) + .or_insert_with(|| CarrierRecord::new(carrier)); + + let mut effects = Vec::new(); + // Canonical content can satisfy a previously latched recovery even if + // the receiver-specific authenticator is invalid. + self.drive_rbc(reference, &mut effects); + + if authentication == IngressAuthentication::Authenticated { + self.carriers + .get_mut(&reference) + .expect("carrier was staged") + .authenticated = true; + let slot_key = (reference.round, reference.authority); + let selected = match self.authenticated_by_slot.get(&slot_key) { + Some(existing) => *existing == reference, + None => { + self.authenticated_by_slot.insert(slot_key, reference); + true + } + }; + if selected && self.in_admission_window(reference.round) { + self.promote_authenticated(reference, &mut effects); + } + } + self.maybe_advance_fast_clock(&mut effects); + self.drain_delivered_phase_batches(&mut effects); + effects + } + + /// Accept an exact recovered carrier only after authenticated phase + /// evidence allocated its candidate. + pub fn recover_carrier( + &mut self, + carrier: CandidateCarrierV1, + ) -> Result, ModelError> { + self.ensure_committee(&carrier)?; + let reference = carrier.reference(); + let key = (reference.round, reference.authority); + let expected = self + .rbc_slots + .get(&key) + .is_some_and(|slot| slot.candidates.contains_key(&reference)); + if !expected { + return Err(ModelError::UnexpectedRecovery(reference)); + } + self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) + } + + /// Record transaction-data availability established by the external + /// Reed-Solomon/reconstruction layer after it verifies this carrier's + /// commitment. Milestone two intentionally treats that layer as a trusted + /// oracle; it does not infer this predicate from unauthenticated ACKs. + pub fn mark_data_available( + &mut self, + reference: BlockReference, + ) -> Result, ModelError> { + self.carriers + .get_mut(&reference) + .ok_or(ModelError::MissingCarrier(reference))? + .data_available = true; + let mut effects = Vec::new(); + self.drive_prefix(reference.authority, &mut effects); + Ok(effects) + } + + pub fn lifecycle(&self, reference: &BlockReference) -> Option { + self.carriers.get(reference).map(CarrierRecord::lifecycle) + } + + pub fn delivered( + &self, + authority: AuthorityIndex, + round: RoundNumber, + ) -> Option { + self.rbc_slots + .get(&(round, authority)) + .and_then(|slot| slot.delivered) + } + + pub fn prefix_tip(&self, authority: AuthorityIndex) -> Option { + let tip = *self.prefix_tips.get(authority as usize)?; + (tip.round > 0).then_some(tip) + } + + pub fn admitted_reference( + &self, + authority: AuthorityIndex, + round: RoundNumber, + ) -> Option { + self.admitted_by_slot.get(&(round, authority)).copied() + } + + /// Include the exact closed-prefix delta named by a committed frontier. + /// The output order is `(round, author, digest)` through `BlockReference`'s + /// canonical ordering. + pub fn apply_frontier( + &mut self, + frontier: &[Option], + ) -> Result, ModelError> { + if frontier.len() != self.committee.len() { + return Err(ModelError::FrontierLength { + expected: self.committee.len(), + actual: frontier.len(), + }); + } + let mut delta = BTreeSet::new(); + for (index, proposed) in frontier.iter().copied().enumerate() { + let authority = index as AuthorityIndex; + if let Some(reference) = proposed { + if reference.authority != authority || reference.round == 0 { + return Err(ModelError::FrontierAuthority { + index: authority, + reference, + }); + } + if !self + .carriers + .get(&reference) + .is_some_and(|record| record.prefix_closed) + { + return Err(ModelError::FrontierNotClosed(reference)); + } + } + let previous = self.included_frontier[index]; + self.collect_frontier_extension(authority, previous, proposed, &mut delta)?; + } + self.included_frontier.clone_from_slice(frontier); + self.included.extend(delta.iter().copied()); + Ok(delta.into_iter().collect()) + } + + fn collect_frontier_extension( + &self, + authority: AuthorityIndex, + previous: Option, + proposed: Option, + delta: &mut BTreeSet, + ) -> Result<(), ModelError> { + let Some(mut cursor) = proposed else { + if previous.is_some() { + return Err(ModelError::FrontierRegression { + authority, + previous, + proposed, + }); + } + return Ok(()); + }; + if Some(cursor) == previous { + return Ok(()); + } + loop { + if Some(cursor) == previous { + return Ok(()); + } + if cursor.round == 0 { + if previous.is_none() && cursor == carrier_genesis_reference(authority) { + return Ok(()); + } + return Err(ModelError::FrontierRegression { + authority, + previous, + proposed, + }); + } + let record = self + .carriers + .get(&cursor) + .ok_or(ModelError::FrontierNotClosed(cursor))?; + if !record.prefix_closed { + return Err(ModelError::FrontierNotClosed(cursor)); + } + delta.insert(cursor); + cursor = record.carrier.header().own_prev(); + } + } + + fn authority_stake(&self, authority: AuthorityIndex) -> Stake { + self.committee.get_stake(authority).unwrap_or(0) + } + + fn ensure_committee(&self, carrier: &CandidateCarrierV1) -> Result<(), ModelError> { + let actual = carrier.committee_id(); + if actual != self.committee_id { + return Err(ModelError::CommitteeMismatch { + expected: self.committee_id, + actual, + }); + } + Ok(()) + } + + fn ensure_authenticated( + &self, + authenticated: &AuthenticatedCarrierV1, + ) -> Result<(), ModelError> { + if authenticated.context() != self.context { + return Err(ModelError::ContextMismatch); + } + if authenticated.receiver() != self.own_authority { + return Err(ModelError::AuthenticationReceiverMismatch { + expected: self.own_authority, + actual: authenticated.receiver(), + }); + } + self.ensure_committee(authenticated.candidate()) + } + + fn ensure_locally_authenticated( + &self, + authenticated: &LocallyAuthenticatedCarrierV1, + ) -> Result<(), ModelError> { + if authenticated.context() != self.context { + return Err(ModelError::ContextMismatch); + } + self.ensure_committee(authenticated.candidate()) + } + + fn voters_stake(&self, voters: &BTreeSet) -> Stake { + voters.iter().fold(0, |stake, authority| { + stake.saturating_add(self.authority_stake(*authority)) + }) + } + + fn rbc_slot_mut(&mut self, reference: BlockReference) -> &mut RbcSlotState { + self.rbc_slots + .entry((reference.round, reference.authority)) + .or_default() + } + + fn authorize_local_echo(&mut self, reference: BlockReference, effects: &mut Vec) { + let own = self.own_authority; + let slot = self.rbc_slot_mut(reference); + if slot.echoed.is_some() { + return; + } + slot.echoed = Some(reference); + slot.echo_by_sender.insert(own, reference); + slot.candidates + .entry(reference) + .or_default() + .echoes + .insert(own); + self.queue_local_phase(RbcPhaseStatementV1::Echo { target: reference }); + self.drive_rbc(reference, effects); + } + + fn queue_local_phase(&mut self, statement: RbcPhaseStatementV1) { + if self.pending_phase_set.insert(statement) { + self.pending_phases.push_back(statement); + } + } + + fn process_phase_batch(&mut self, outer: BlockReference, effects: &mut Vec) { + self.process_phase_batch_steps(outer, usize::MAX, effects); + self.drain_delivered_phase_batches(effects); + } + + fn drain_delivered_phase_batches(&mut self, effects: &mut Vec) { + while let Some(outer) = self.pending_delivered_batch_replays.pop_front() { + self.process_phase_batch_steps(outer, usize::MAX, effects); + } + } + + fn process_phase_batch_steps( + &mut self, + outer: BlockReference, + maximum_steps: usize, + effects: &mut Vec, + ) { + let mut processed = 0; + loop { + if processed == maximum_steps { + return; + } + let Some((sender, statement)) = self.carriers.get(&outer).and_then(|record| { + record + .carrier + .header() + .phase_batch() + .get(record.phase_batch_cursor) + .copied() + .map(|statement| (record.carrier.header().author(), statement)) + }) else { + return; + }; + // Applying the statement is idempotent. Advance the persisted + // cursor only afterwards, so a crash between the two replays the + // same statement rather than skipping the unprocessed tail. + self.record_phase(sender, statement, effects); + self.carriers + .get_mut(&outer) + .expect("the outer carrier remains pinned") + .phase_batch_cursor += 1; + processed += 1; + } + } + + fn record_phase( + &mut self, + sender: AuthorityIndex, + statement: RbcPhaseStatementV1, + effects: &mut Vec, + ) { + if !self.committee.known_authority(sender) { + return; + } + let target = statement.target(); + if sender == self.own_authority { + let authorized = self + .rbc_slots + .get(&(target.round, target.authority)) + .is_some_and(|slot| match statement { + RbcPhaseStatementV1::Echo { .. } => slot.echoed == Some(target), + RbcPhaseStatementV1::Ready { .. } => slot.readied == Some(target), + }); + if !authorized { + // An own-authored embedded statement is replay, not fresh + // authority. The corresponding persisted local lock must + // already exist before it may reconstruct sender evidence. + return; + } + } + let slot = self.rbc_slot_mut(target); + let senders = match statement { + RbcPhaseStatementV1::Echo { .. } => &mut slot.echo_by_sender, + RbcPhaseStatementV1::Ready { .. } => &mut slot.ready_by_sender, + }; + match senders.get(&sender) { + Some(existing) if *existing != target => return, + Some(_) => return, + None => { + senders.insert(sender, target); + } + } + let candidate = slot.candidates.entry(target).or_default(); + match statement { + RbcPhaseStatementV1::Echo { .. } => { + candidate.echoes.insert(sender); + } + RbcPhaseStatementV1::Ready { .. } => { + candidate.readies.insert(sender); + } + } + self.drive_rbc(target, effects); + } + + fn drive_rbc(&mut self, target: BlockReference, effects: &mut Vec) { + let slot_key = (target.round, target.authority); + if !self + .rbc_slots + .get(&slot_key) + .is_some_and(|slot| slot.candidates.contains_key(&target)) + { + // Merely staging canonical content is not authenticated RBC + // evidence. Candidate state is allocated only by a locally + // authorized ECHO or an embedded ECHO/READY statement. + return; + } + loop { + let header_available = self.carriers.contains_key(&target); + let q = self.committee.quorum_threshold(); + let v = self.committee.validity_threshold(); + let action = { + let echo_stake; + let ready_stake; + { + let slot = self.rbc_slot_mut(target); + let candidate = slot.candidates.entry(target).or_default(); + echo_stake = candidate.echoes.clone(); + ready_stake = candidate.readies.clone(); + } + let echo_stake = self.voters_stake(&echo_stake); + let ready_stake = self.voters_stake(&ready_stake); + let slot = self.rbc_slot_mut(target); + let candidate = slot.candidates.entry(target).or_default(); + candidate.echo_quorum_observed |= echo_stake >= q; + candidate.ready_validity_observed |= ready_stake >= v; + candidate.ready_quorum_observed |= ready_stake >= q; + let ready_trigger = + candidate.echo_quorum_observed || candidate.ready_validity_observed; + let needs_header = !header_available + && ((slot.readied.is_none() && ready_trigger) + || (slot.delivered.is_none() && candidate.ready_quorum_observed)); + if needs_header { + let holders = candidate.holders(); + if holders != candidate.requested_holders { + candidate.requested_holders = holders; + RbcAction::NeedCarrier + } else { + RbcAction::None + } + } else if header_available && slot.readied.is_none() && ready_trigger { + RbcAction::SendReady + } else if header_available + && slot.delivered.is_none() + && candidate.ready_quorum_observed + { + RbcAction::Deliver + } else { + RbcAction::None + } + }; + + match action { + RbcAction::NeedCarrier => { + let holders = self + .rbc_slots + .get(&(target.round, target.authority)) + .and_then(|slot| slot.candidates.get(&target)) + .map(RbcCandidateState::holders) + .unwrap_or_default() + .into_iter() + .collect(); + effects.push(ModelEffect::NeedCarrier { target, holders }); + break; + } + RbcAction::SendReady => { + let own = self.own_authority; + let slot = self.rbc_slot_mut(target); + slot.readied = Some(target); + slot.ready_by_sender.insert(own, target); + slot.candidates + .entry(target) + .or_default() + .readies + .insert(own); + self.queue_local_phase(RbcPhaseStatementV1::Ready { target }); + } + RbcAction::Deliver => { + self.rbc_slot_mut(target).delivered = Some(target); + let record = self + .carriers + .get_mut(&target) + .expect("delivery requires exact canonical carrier content"); + record.delivered = true; + effects.push(ModelEffect::Delivered(target)); + self.pending_delivered_batch_replays.push_back(target); + self.drive_prefix(target.authority, effects); + } + RbcAction::None => break, + } + } + } + + fn maybe_advance_fast_clock(&mut self, effects: &mut Vec) { + let round = self.local_carrier_round; + if !self.own_fixed.contains_key(&round) { + return; + } + let admitted: BTreeSet<_> = self + .admitted_by_slot + .keys() + .filter_map(|(candidate_round, authority)| { + (*candidate_round == round).then_some(*authority) + }) + .collect(); + if self.voters_stake(&admitted) < self.committee.quorum_threshold() { + return; + } + self.local_carrier_round = round.saturating_add(1); + effects.push(ModelEffect::CarrierRoundAdvanced(self.local_carrier_round)); + self.promote_buffered_window(effects); + } + + fn in_admission_window(&self, round: RoundNumber) -> bool { + round + <= self + .local_carrier_round + .saturating_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) + } + + fn promote_authenticated(&mut self, reference: BlockReference, effects: &mut Vec) { + let slot_key = (reference.round, reference.authority); + if self.authenticated_by_slot.get(&slot_key) != Some(&reference) + || self + .carriers + .get(&reference) + .is_none_or(|record| record.admitted) + { + return; + } + self.admitted_by_slot.insert(slot_key, reference); + self.carriers + .get_mut(&reference) + .expect("an authenticated carrier remains staged") + .admitted = true; + self.authorize_local_echo(reference, effects); + self.process_phase_batch(reference, effects); + } + + fn promote_buffered_window(&mut self, effects: &mut Vec) { + let eligible: Vec<_> = self + .authenticated_by_slot + .values() + .copied() + .filter(|reference| self.in_admission_window(reference.round)) + .collect(); + for reference in eligible { + self.promote_authenticated(reference, effects); + } + } + + fn drive_prefix(&mut self, authority: AuthorityIndex, effects: &mut Vec) { + loop { + let Some(current_tip) = self.prefix_tips.get(authority as usize).copied() else { + return; + }; + let Some(next_round) = current_tip.round.checked_add(1) else { + return; + }; + let Some(next) = self.delivered(authority, next_round) else { + return; + }; + let can_close = self.carriers.get(&next).is_some_and(|record| { + record.delivered + && record.data_available + && record.carrier.header().own_prev() == current_tip + }); + if !can_close { + return; + } + self.carriers + .get_mut(&next) + .expect("delivered carrier exists") + .prefix_closed = true; + self.prefix_tips[authority as usize] = next; + effects.push(ModelEffect::PrefixAdvanced { + authority, + tip: next, + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + crypto::{TransactionsCommitment, mac_keyrings_for_test}, + starfish_rbc_dag::{CarrierHeaderV1Args, RbcDagError}, + types::{BlockAuthenticationScheme, BlockReference}, + }; + + fn committee(n: usize) -> Arc { + Committee::new_test(vec![1; n]) + } + + fn context(committee: &Committee) -> RbcDagContextV1 { + RbcDagContextV1::new( + super::super::RbcDagProtocolInstanceId::new([0xD1; 32]).unwrap(), + committee, + BlockAuthenticationScheme::MacVector, + ) + .unwrap() + } + + fn model(committee: Arc, authority: AuthorityIndex) -> RbcDagModel { + let context = context(&committee); + RbcDagModel::new(committee, authority, context).unwrap() + } + + fn authenticate_for( + committee: &Committee, + carrier: &CandidateCarrierV1, + receiver: AuthorityIndex, + ) -> AuthenticatedCarrierV1 { + let context = context(committee); + let keyrings = mac_keyrings_for_test(committee.len()); + let author = carrier.header().author() as usize; + let authentication = context + .authenticate( + carrier, + committee, + super::super::CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ) + .unwrap(); + context + .verify_authentication( + carrier.clone(), + authentication, + receiver, + committee, + &keyrings[receiver as usize], + ) + .unwrap() + } + + fn authenticate_local( + committee: &Committee, + carrier: &CandidateCarrierV1, + ) -> LocallyAuthenticatedCarrierV1 { + let context = context(committee); + let keyrings = mac_keyrings_for_test(committee.len()); + let author = carrier.header().author() as usize; + context + .authenticate_local( + carrier.clone(), + committee, + super::super::CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ) + .unwrap() + } + + fn admit( + model: &mut RbcDagModel, + carrier: CandidateCarrierV1, + ) -> Result, ModelError> { + let authenticated = authenticate_for(&model.committee, &carrier, model.own_authority); + model.receive_authenticated(authenticated) + } + + fn candidate( + committee: &Committee, + author: AuthorityIndex, + round: RoundNumber, + own_prev: BlockReference, + weak_parents: Vec, + phase_batch: Vec, + marker: u64, + ) -> Result { + CandidateCarrierV1::try_new( + CarrierHeaderV1Args { + author, + carrier_round: round, + own_prev, + weak_parents, + transactions_commitment: TransactionsCommitment::default(), + data_acknowledgments: Vec::new(), + phase_batch, + consensus_vertex: None, + creation_time_ns: marker, + }, + committee, + ) + } + + fn genesis_parents( + committee: &Committee, + author: AuthorityIndex, + ) -> (BlockReference, Vec) { + let own = carrier_genesis_reference(author); + let mut stake = committee.get_stake(author).unwrap(); + let mut weak = Vec::new(); + for other in committee.authorities() { + if other == author { + continue; + } + weak.push(carrier_genesis_reference(other)); + stake += committee.get_stake(other).unwrap(); + if stake >= committee.quorum_threshold() { + break; + } + } + (own, weak) + } + + fn build_local(model: &mut RbcDagModel, marker: u64) -> CandidateCarrierV1 { + let (own_prev, weak_parents) = model.local_parent_set().unwrap(); + let carrier = candidate( + &model.committee, + model.own_authority, + model.local_carrier_round, + own_prev, + weak_parents, + model.pending_phase_batch(), + marker, + ) + .unwrap(); + let authenticated = authenticate_local(&model.committee, &carrier); + model.start_local_carrier(authenticated).unwrap(); + carrier + } + + fn run_honest_round(models: &mut [RbcDagModel], round: RoundNumber) -> Vec { + let carriers: Vec<_> = models + .iter_mut() + .enumerate() + .map(|(index, model)| build_local(model, u64::from(round) * 100 + index as u64)) + .collect(); + for carrier in &carriers { + for model in models.iter_mut() { + if model.own_authority != carrier.header().author() { + let authenticated = + authenticate_for(&model.committee, carrier, model.own_authority); + model.receive_authenticated(authenticated).unwrap(); + } + model.mark_data_available(carrier.reference()).unwrap(); + } + } + assert!( + models + .iter() + .all(|model| model.local_carrier_round() == round + 1) + ); + carriers + } + + fn all_honest_progress(n: usize) { + let committee = committee(n); + let mut models: Vec<_> = committee + .authorities() + .map(|authority| model(Arc::clone(&committee), authority)) + .collect(); + let mut rounds = Vec::new(); + for round in 1..=6 { + rounds.push(run_honest_round(&mut models, round)); + } + for model in &models { + for carriers in rounds.iter().take(4) { + for carrier in carriers { + assert_eq!( + model + .delivered(carrier.header().author(), carrier.header().carrier_round()), + Some(carrier.reference()) + ); + assert!(model.lifecycle(&carrier.reference()).unwrap().prefix_closed); + } + } + } + } + + #[test] + fn four_node_heartbeat_only_run_delivers_every_mature_carrier() { + all_honest_progress(4); + } + + #[test] + fn seven_node_heartbeat_only_run_delivers_every_mature_carrier() { + all_honest_progress(7); + } + + #[test] + fn phase_backlog_exposes_only_a_bounded_fifo_prefix() { + let committee = committee(4); + let mut model = model(committee, 0); + model.local_carrier_round = 4; + let mut queued = Vec::new(); + for round in 1..=3 { + for author in 0..4 { + let target = BlockReference::new_test(author, round); + queued.push(RbcPhaseStatementV1::Echo { target }); + queued.push(RbcPhaseStatementV1::Ready { target }); + } + } + for statement in &queued { + model.queue_local_phase(*statement); + } + + assert_eq!(model.pending_phase_backlog_len(), 24); + assert_eq!(model.pending_phase_batch(), queued[..16]); + } + + #[test] + fn local_carrier_must_drain_the_exact_bounded_phase_prefix() { + let committee = committee(4); + let mut models: Vec<_> = (0..4) + .map(|authority| model(Arc::clone(&committee), authority)) + .collect(); + run_honest_round(&mut models, 1); + + let model = &mut models[0]; + let expected = model.pending_phase_batch(); + assert!(!expected.is_empty()); + let (own_prev, weak_parents) = model.local_parent_set().unwrap(); + let omitted = candidate( + &committee, + model.own_authority, + model.local_carrier_round, + own_prev, + weak_parents.clone(), + Vec::new(), + 0xA1, + ) + .unwrap(); + let omitted_authentication = authenticate_local(&committee, &omitted); + assert_eq!( + model.start_local_carrier(omitted_authentication), + Err(ModelError::LocalPhaseBatchMismatch) + ); + assert!(model.can_create_carrier()); + + let exact = candidate( + &committee, + model.own_authority, + model.local_carrier_round, + own_prev, + weak_parents, + expected, + 0xA2, + ) + .unwrap(); + let exact_authentication = authenticate_local(&committee, &exact); + model.start_local_carrier(exact_authentication).unwrap(); + assert_eq!(model.own_fixed.get(&2), Some(&exact.reference())); + } + + fn record_phase( + model: &mut RbcDagModel, + sender: AuthorityIndex, + statement: RbcPhaseStatementV1, + ) -> Vec { + let mut effects = Vec::new(); + model.record_phase(sender, statement, &mut effects); + model.drain_delivered_phase_batches(&mut effects); + effects + } + + fn force_deliver(model: &mut RbcDagModel, carrier: CandidateCarrierV1) { + let target = carrier.reference(); + model.stage_candidate(carrier).unwrap(); + let senders: Vec<_> = model + .committee + .authorities() + .filter(|sender| *sender != model.own_authority) + .take(model.committee.quorum_threshold() as usize) + .collect(); + for sender in senders { + record_phase(model, sender, RbcPhaseStatementV1::Ready { target }); + } + assert_eq!( + model.delivered(target.authority, target.round), + Some(target) + ); + } + + #[test] + fn threshold_before_header_requests_then_recovers_exact_carrier() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 1).unwrap(); + let target = carrier.reference(); + + assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }).is_empty()); + assert!(record_phase(&mut model, 1, RbcPhaseStatementV1::Echo { target }).is_empty()); + assert!(matches!( + record_phase( + &mut model, + 2, + RbcPhaseStatementV1::Echo { target } + ) + .as_slice(), + [ModelEffect::NeedCarrier { target: requested, holders }] + if *requested == target && holders == &[0, 1, 2] + )); + + model.recover_carrier(carrier).unwrap(); + assert!( + model + .pending_phases + .contains(&RbcPhaseStatementV1::Ready { target }) + ); + let lifecycle = model.lifecycle(&target).unwrap(); + assert!(!lifecycle.authenticated); + assert!(!lifecycle.admitted); + } + + #[test] + fn staged_content_without_phase_evidence_cannot_authorize_recovery() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 2).unwrap(); + let reference = carrier.reference(); + + model.stage_candidate(carrier.clone()).unwrap(); + assert!(model.lifecycle(&reference).is_some()); + assert!(model.rbc_slots.is_empty()); + assert_eq!( + model.recover_carrier(carrier), + Err(ModelError::UnexpectedRecovery(reference)) + ); + assert!(model.rbc_slots.is_empty()); + } + + #[test] + fn ready_threshold_before_content_recovers_then_delivers() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 3).unwrap(); + let target = carrier.reference(); + + assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target }).is_empty()); + assert!(matches!( + record_phase( + &mut model, + 1, + RbcPhaseStatementV1::Ready { target } + ) + .as_slice(), + [ModelEffect::NeedCarrier { target: requested, holders }] + if *requested == target && holders == &[0, 1] + )); + assert_eq!(model.delivered(0, 1), None); + assert!(model.lifecycle(&target).is_none()); + + let effects = model.recover_carrier(carrier).unwrap(); + assert!(effects.contains(&ModelEffect::Delivered(target))); + assert_eq!(model.delivered(0, 1), Some(target)); + assert!( + model + .pending_phases + .contains(&RbcPhaseStatementV1::Ready { target }) + ); + } + + #[test] + fn cross_committee_candidate_is_rejected_before_state_mutation() { + let local_committee = committee(4); + let foreign_committee = Committee::new_test(vec![2; 4]); + let mut model = model(Arc::clone(&local_committee), 3); + let (own_prev, weak) = genesis_parents(&foreign_committee, 0); + let foreign = candidate(&foreign_committee, 0, 1, own_prev, weak, Vec::new(), 9).unwrap(); + let reference = foreign.reference(); + + assert!(matches!( + model.stage_candidate(foreign), + Err(ModelError::CommitteeMismatch { .. }) + )); + assert!(model.lifecycle(&reference).is_none()); + assert!(model.admitted_reference(0, 1).is_none()); + assert!(model.rbc_slots.is_empty()); + } + + #[test] + fn authenticated_capability_is_bound_to_context_and_receiver() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 10).unwrap(); + let reference = carrier.reference(); + + let wrong_receiver = authenticate_for(&committee, &carrier, 2); + assert_eq!( + model.receive_authenticated(wrong_receiver), + Err(ModelError::AuthenticationReceiverMismatch { + expected: 3, + actual: 2, + }) + ); + + let other_context = RbcDagContextV1::new( + super::super::RbcDagProtocolInstanceId::new([0xD2; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ) + .unwrap(); + let keyrings = mac_keyrings_for_test(committee.len()); + let authentication = other_context + .authenticate( + &carrier, + &committee, + super::super::CarrierAuthorizerV1::MacVector { + authority: 0, + keys: &keyrings[0], + }, + ) + .unwrap(); + let wrong_context = other_context + .verify_authentication(carrier, authentication, 3, &committee, &keyrings[3]) + .unwrap(); + assert_eq!( + model.receive_authenticated(wrong_context), + Err(ModelError::ContextMismatch) + ); + assert!(model.lifecycle(&reference).is_none()); + assert!(model.rbc_slots.is_empty()); + } + + #[test] + fn locally_authored_carrier_can_only_enter_through_atomic_start() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 0); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 11).unwrap(); + let reference = carrier.reference(); + let authenticated = authenticate_for(&committee, &carrier, 0); + + assert_eq!( + model.receive_authenticated(authenticated), + Err(ModelError::LocalCarrierRequiresStart(reference)) + ); + assert!(model.lifecycle(&reference).is_none()); + assert!(model.own_fixed.is_empty()); + assert!(model.rbc_slots.is_empty()); + } + + #[test] + fn own_embedded_phase_requires_the_persisted_local_lock() { + let committee = committee(4); + let mut model = model(committee, 3); + let target = BlockReference::new_test(0, 1); + + assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Echo { target }).is_empty()); + assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Ready { target }).is_empty()); + assert!(model.rbc_slots.is_empty()); + } + + #[test] + fn phase_replay_and_equivocation_count_each_sender_once() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let first = BlockReference::new_test(0, 1); + let conflicting = BlockReference::new_test(0, 1); + let mut conflicting = conflicting; + conflicting.digest = crate::types::BlockDigest::from([0x77; 32]); + + for _ in 0..3 { + assert!( + record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target: first }).is_empty() + ); + } + assert!( + record_phase( + &mut model, + 0, + RbcPhaseStatementV1::Echo { + target: conflicting + } + ) + .is_empty() + ); + assert!( + record_phase(&mut model, 1, RbcPhaseStatementV1::Echo { target: first }).is_empty() + ); + assert!(matches!( + record_phase( + &mut model, + 2, + RbcPhaseStatementV1::Echo { target: first } + ) + .as_slice(), + [ModelEffect::NeedCarrier { target, .. }] if *target == first + )); + let slot = model.rbc_slots.get(&(1, 0)).unwrap(); + assert_eq!(slot.echo_by_sender.len(), 3); + assert_eq!(slot.echo_by_sender[&0], first); + assert!(!slot.candidates.contains_key(&conflicting)); + + record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target: first }); + record_phase( + &mut model, + 0, + RbcPhaseStatementV1::Ready { + target: conflicting, + }, + ); + let slot = model.rbc_slots.get(&(1, 0)).unwrap(); + assert_eq!(slot.ready_by_sender[&0], first); + assert!(!slot.candidates.contains_key(&conflicting)); + } + + #[test] + fn split_initial_values_converge_on_one_delivery() { + let committee = committee(4); + let (own_prev, weak) = genesis_parents(&committee, 0); + let first = candidate(&committee, 0, 1, own_prev, weak.clone(), Vec::new(), 11).unwrap(); + let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 12).unwrap(); + let first_ref = first.reference(); + let conflicting_ref = conflicting.reference(); + let mut models: Vec<_> = committee + .authorities() + .map(|authority| model(Arc::clone(&committee), authority)) + .collect(); + + for (index, model) in models.iter_mut().enumerate() { + let (admitted, staged) = if index == 3 { + (conflicting.clone(), first.clone()) + } else { + (first.clone(), conflicting.clone()) + }; + if model.own_authority == admitted.header().author() { + // The dealer is Byzantine in this trace. Its local behavior is + // outside the honest local-start API, so retain both bytes and + // drive its receive-side RBC state only from phase evidence. + model.stage_candidate(admitted).unwrap(); + model.stage_candidate(staged).unwrap(); + } else { + admit(model, admitted).unwrap(); + model.stage_candidate(staged).unwrap(); + } + } + + // Authorities 0, 1, and 2 ECHO the first value; authority 3 ECHOs the + // conflicting value. The first value reaches Q and READY amplification + // carries the receiver that saw the split INIT to the same delivery. + for model in &mut models { + for sender in 0..3 { + record_phase( + model, + sender, + RbcPhaseStatementV1::Echo { target: first_ref }, + ); + } + record_phase( + model, + 3, + RbcPhaseStatementV1::Echo { + target: conflicting_ref, + }, + ); + } + for model in &mut models { + for sender in 0..3 { + record_phase( + model, + sender, + RbcPhaseStatementV1::Ready { target: first_ref }, + ); + } + } + assert!( + models + .iter() + .all(|model| model.delivered(0, 1) == Some(first_ref)) + ); + assert!( + models + .iter() + .all(|model| model.delivered(0, 1) != Some(conflicting_ref)) + ); + } + + #[test] + fn typed_outer_carriers_enforce_split_value_phase_locks_end_to_end() { + let committee = committee(4); + let (own_prev, weak) = genesis_parents(&committee, 0); + let first = candidate(&committee, 0, 1, own_prev, weak.clone(), Vec::new(), 21).unwrap(); + let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 22).unwrap(); + let first_ref = first.reference(); + let conflicting_ref = conflicting.reference(); + let mut receiver = model(Arc::clone(&committee), 3); + + admit(&mut receiver, first).unwrap(); + admit(&mut receiver, conflicting).unwrap(); + assert_eq!(receiver.admitted_reference(0, 1), Some(first_ref)); + + let outer = |author: AuthorityIndex, + round: RoundNumber, + phase_batch: Vec, + marker: u64| { + let own_prev = BlockReference::new_test(author, round - 1); + let weak_parents = (0..4) + .filter(|other| *other != author) + .take(2) + .map(|other| BlockReference::new_test(other, round - 1)) + .collect(); + candidate( + &committee, + author, + round, + own_prev, + weak_parents, + phase_batch, + marker, + ) + .unwrap() + }; + + for author in 0..3 { + let carrier = outer( + author, + 2, + vec![RbcPhaseStatementV1::Echo { target: first_ref }], + 200 + u64::from(author), + ); + admit(&mut receiver, carrier).unwrap(); + } + for author in 0..3 { + let mut phase_batch = Vec::new(); + if author == 0 { + phase_batch.push(RbcPhaseStatementV1::Echo { + target: conflicting_ref, + }); + } + phase_batch.push(RbcPhaseStatementV1::Ready { target: first_ref }); + let carrier = outer(author, 3, phase_batch, 300 + u64::from(author)); + admit(&mut receiver, carrier).unwrap(); + } + + assert_eq!(receiver.delivered(0, 1), Some(first_ref)); + assert_ne!(receiver.delivered(0, 1), Some(conflicting_ref)); + let slot = receiver.rbc_slots.get(&(1, 0)).unwrap(); + assert_eq!(slot.echo_by_sender.get(&0), Some(&first_ref)); + assert!(!slot.candidates.contains_key(&conflicting_ref)); + } + + #[test] + fn non_equivocating_phase_reordering_has_the_same_result() { + let committee = committee(4); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 15).unwrap(); + let target = carrier.reference(); + let mut left = model(Arc::clone(&committee), 3); + let mut right = model(Arc::clone(&committee), 3); + left.stage_candidate(carrier.clone()).unwrap(); + right.stage_candidate(carrier).unwrap(); + + for sender in [0, 1, 2] { + record_phase(&mut left, sender, RbcPhaseStatementV1::Echo { target }); + } + for sender in [2, 0, 1] { + record_phase(&mut right, sender, RbcPhaseStatementV1::Echo { target }); + } + for sender in [0, 1, 2] { + record_phase(&mut left, sender, RbcPhaseStatementV1::Ready { target }); + } + for sender in [1, 2, 0] { + record_phase(&mut right, sender, RbcPhaseStatementV1::Ready { target }); + } + assert_eq!(left.delivered(0, 1), Some(target)); + assert_eq!(right.delivered(0, 1), Some(target)); + assert_eq!(left.pending_phase_batch(), right.pending_phase_batch()); + } + + #[test] + fn delivered_candidate_replays_batch_even_without_admission() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let target = BlockReference::new_test(1, 1); + let own_prev = BlockReference::new_test(0, 1); + let weak = vec![ + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ]; + let outer = candidate( + &committee, + 0, + 2, + own_prev, + weak, + vec![RbcPhaseStatementV1::Echo { target }], + 2, + ) + .unwrap(); + let outer_ref = outer.reference(); + model.stage_candidate(outer.clone()).unwrap(); + assert!( + !model + .rbc_slots + .get(&(target.round, target.authority)) + .is_some_and(|slot| slot.echo_by_sender.contains_key(&0)) + ); + + force_deliver(&mut model, outer); + assert_eq!( + model.rbc_slots[&(target.round, target.authority)].echo_by_sender[&0], + target + ); + let lifecycle = model.lifecycle(&outer_ref).unwrap(); + assert!(!lifecycle.admitted); + assert!(lifecycle.delivered); + assert!(lifecycle.phase_batch_processed); + } + + #[test] + fn replay_after_crash_before_batch_cursor_does_not_skip_the_tail() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let own_prev = BlockReference::new_test(0, 1); + let weak = vec![ + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ]; + let first = BlockReference::new_test(1, 1); + let second = BlockReference::new_test(2, 1); + let outer = candidate( + &committee, + 0, + 2, + own_prev, + weak, + vec![ + RbcPhaseStatementV1::Echo { target: first }, + RbcPhaseStatementV1::Ready { target: second }, + ], + 0xCA, + ) + .unwrap(); + let outer_ref = outer.reference(); + model.stage_candidate(outer).unwrap(); + + let mut uninterrupted = model.clone(); + uninterrupted.process_phase_batch(outer_ref, &mut Vec::new()); + + // Model a crash after the first idempotent statement was persisted but + // before the outer batch cursor was advanced. + let mut restarted = model; + restarted.record_phase( + 0, + RbcPhaseStatementV1::Echo { target: first }, + &mut Vec::new(), + ); + restarted.process_phase_batch(outer_ref, &mut Vec::new()); + + assert_eq!(restarted.rbc_slots, uninterrupted.rbc_slots); + assert_eq!(restarted.pending_phases, uninterrupted.pending_phases); + assert!( + restarted + .lifecycle(&outer_ref) + .unwrap() + .phase_batch_processed + ); + } + + #[test] + fn deeply_chained_delivered_batches_use_an_explicit_work_queue() { + const DEPTH: RoundNumber = 2_048; + + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let mut previous = None; + let mut references = Vec::with_capacity(DEPTH as usize); + for round in 1..=DEPTH { + let own_prev = previous.unwrap_or_else(|| carrier_genesis_reference(0)); + let parent = |authority| { + if round == 1 { + carrier_genesis_reference(authority) + } else { + BlockReference::new_test(authority, round - 1) + } + }; + let phase_batch = previous + .map(|target| vec![RbcPhaseStatementV1::Ready { target }]) + .unwrap_or_default(); + let carrier = candidate( + &committee, + 0, + round, + own_prev, + vec![parent(1), parent(2)], + phase_batch, + u64::from(round), + ) + .unwrap(); + let reference = carrier.reference(); + model + .carriers + .insert(reference, CarrierRecord::new(carrier)); + let mut candidate_state = RbcCandidateState::default(); + candidate_state.readies.extend([1, 2]); + let mut slot = RbcSlotState::default(); + slot.ready_by_sender + .extend([(1, reference), (2, reference)]); + slot.candidates.insert(reference, candidate_state); + model.rbc_slots.insert((round, 0), slot); + references.push(reference); + previous = Some(reference); + } + + model + .pending_delivered_batch_replays + .push_back(*references.last().unwrap()); + let mut effects = Vec::new(); + model.drain_delivered_phase_batches(&mut effects); + + assert!(model.pending_delivered_batch_replays.is_empty()); + assert_eq!(model.delivered(0, 1), Some(references[0])); + assert_eq!( + model.delivered(0, DEPTH - 1), + Some(references[DEPTH as usize - 2]) + ); + assert!(model.delivered(0, DEPTH).is_none()); + } + + #[test] + fn quorum_of_future_carriers_cannot_jump_the_sequential_clock() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 0); + let mut future_references = Vec::new(); + for author in 1..4 { + let own_prev = BlockReference::new_test(author, 4); + let weak = (0..4) + .filter(|other| *other != author) + .take(2) + .map(|other| BlockReference::new_test(other, 4)) + .collect(); + let future = candidate( + &committee, + author, + 5, + own_prev, + weak, + Vec::new(), + u64::from(author), + ) + .unwrap(); + future_references.push(future.reference()); + admit(&mut model, future).unwrap(); + } + assert_eq!(model.local_carrier_round(), 1); + assert!(model.can_create_carrier()); + for reference in future_references { + let lifecycle = model.lifecycle(&reference).unwrap(); + assert!(lifecycle.authenticated); + assert!(!lifecycle.admitted); + assert!( + !model + .rbc_slots + .contains_key(&(reference.round, reference.authority)) + ); + } + } + + #[test] + fn carrier_beyond_the_bounded_future_buffer_is_rejected_without_state() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 0); + let carrier = candidate( + &committee, + 1, + 6, + BlockReference::new_test(1, 5), + vec![ + BlockReference::new_test(0, 5), + BlockReference::new_test(2, 5), + ], + Vec::new(), + 60, + ) + .unwrap(); + let reference = carrier.reference(); + let authenticated = authenticate_for(&committee, &carrier, 0); + + assert_eq!( + model.receive_authenticated(authenticated), + Err(ModelError::FutureCarrierOutsideBuffer { + current: 1, + maximum: 5, + actual: 6, + }) + ); + assert!(model.lifecycle(&reference).is_none()); + assert!(model.authenticated_by_slot.is_empty()); + assert!(model.rbc_slots.is_empty()); + } + + #[test] + fn buffered_authenticated_carrier_is_promoted_when_window_opens() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 0); + let future = candidate( + &committee, + 1, + 4, + BlockReference::new_test(1, 3), + vec![ + BlockReference::new_test(0, 3), + BlockReference::new_test(2, 3), + ], + Vec::new(), + 40, + ) + .unwrap(); + let future_ref = future.reference(); + admit(&mut model, future).unwrap(); + assert!(model.lifecycle(&future_ref).unwrap().authenticated); + assert!(!model.lifecycle(&future_ref).unwrap().admitted); + + build_local(&mut model, 1); + for author in [1, 2] { + let (own_prev, weak) = genesis_parents(&committee, author); + let round_one = candidate( + &committee, + author, + 1, + own_prev, + weak, + Vec::new(), + u64::from(author), + ) + .unwrap(); + admit(&mut model, round_one).unwrap(); + } + + assert_eq!(model.local_carrier_round(), 2); + assert!(model.lifecycle(&future_ref).unwrap().admitted); + assert_eq!(model.admitted_reference(1, 4), Some(future_ref)); + + let future_echo = RbcPhaseStatementV1::Echo { target: future_ref }; + for round in 2..=4 { + assert_eq!(model.local_carrier_round(), round); + assert!(!model.pending_phase_batch().contains(&future_echo)); + build_local(&mut model, 100 + u64::from(round)); + assert!(model.pending_phases.contains(&future_echo)); + for author in [2, 3] { + let own_prev = BlockReference::new_test(author, round - 1); + let weak = (0..4) + .filter(|other| *other != author) + .take(2) + .map(|other| BlockReference::new_test(other, round - 1)) + .collect(); + let remote = candidate( + &committee, + author, + round, + own_prev, + weak, + Vec::new(), + u64::from(round) * 10 + u64::from(author), + ) + .unwrap(); + admit(&mut model, remote).unwrap(); + } + assert_eq!(model.local_carrier_round(), round + 1); + } + + assert!(model.pending_phase_batch().contains(&future_echo)); + let round_five = build_local(&mut model, 500); + assert!(round_five.header().phase_batch().contains(&future_echo)); + assert!(!model.pending_phases.contains(&future_echo)); + } + + #[test] + fn poisoned_recipient_still_delivers_without_optimistic_admission() { + let committee = committee(4); + let mut models: Vec<_> = committee + .authorities() + .map(|authority| model(Arc::clone(&committee), authority)) + .collect(); + let round_one: Vec<_> = models + .iter_mut() + .enumerate() + .map(|(index, model)| build_local(model, index as u64)) + .collect(); + let poisoned = round_one[0].reference(); + for carrier in &round_one { + for model in &mut models { + if model.own_authority == carrier.header().author() { + continue; + } + if carrier.reference() == poisoned && model.own_authority == 3 { + model.stage_candidate(carrier.clone()).unwrap(); + } else { + admit(model, carrier.clone()).unwrap(); + } + } + } + assert_ne!(models[3].admitted_reference(0, 1), Some(poisoned)); + for round in 2..=4 { + run_honest_round(&mut models, round); + } + for model in &models { + assert_eq!(model.delivered(0, 1), Some(poisoned)); + } + let poisoned_lifecycle = models[3].lifecycle(&poisoned).unwrap(); + assert!(!poisoned_lifecycle.admitted); + assert!(poisoned_lifecycle.delivered); + } + + #[test] + fn missing_weak_parent_body_does_not_trigger_fetch_or_block_delivery() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (g0, weak0) = genesis_parents(&committee, 0); + let round_one = candidate(&committee, 0, 1, g0, weak0, Vec::new(), 1).unwrap(); + let round_one_ref = round_one.reference(); + force_deliver(&mut model, round_one); + model.mark_data_available(round_one_ref).unwrap(); + + let missing = BlockReference::new_test(1, 1); + let round_two = candidate( + &committee, + 0, + 2, + round_one_ref, + vec![missing, BlockReference::new_test(2, 1)], + Vec::new(), + 2, + ) + .unwrap(); + let round_two_ref = round_two.reference(); + let effects = admit(&mut model, round_two.clone()).unwrap(); + assert!(!effects.iter().any( + |effect| matches!(effect, ModelEffect::NeedCarrier { target, .. } if *target == missing) + )); + force_deliver(&mut model, round_two); + model.mark_data_available(round_two_ref).unwrap(); + assert_eq!(model.prefix_tip(0), Some(round_two_ref)); + } + + #[test] + fn f_missing_weak_parent_bodies_do_not_block_seven_node_delivery() { + let committee = committee(7); + let mut model = model(Arc::clone(&committee), 6); + let (genesis, weak) = genesis_parents(&committee, 0); + let first = candidate(&committee, 0, 1, genesis, weak, Vec::new(), 1).unwrap(); + let first_ref = first.reference(); + force_deliver(&mut model, first); + model.mark_data_available(first_ref).unwrap(); + + let mut known = Vec::new(); + for author in [3, 4] { + let (own_prev, weak) = genesis_parents(&committee, author); + let carrier = candidate( + &committee, + author, + 1, + own_prev, + weak, + Vec::new(), + u64::from(author), + ) + .unwrap(); + known.push(carrier.reference()); + model.stage_candidate(carrier).unwrap(); + } + let missing = [ + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ]; + let second = candidate( + &committee, + 0, + 2, + first_ref, + vec![missing[0], missing[1], known[0], known[1]], + Vec::new(), + 2, + ) + .unwrap(); + let second_ref = second.reference(); + let effects = admit(&mut model, second.clone()).unwrap(); + assert!(effects.iter().all( + |effect| !matches!(effect, ModelEffect::NeedCarrier { target, .. } if missing.contains(target)) + )); + assert!( + missing + .iter() + .all(|reference| model.lifecycle(reference).is_none()) + ); + + force_deliver(&mut model, second); + model.mark_data_available(second_ref).unwrap(); + assert_eq!(model.prefix_tip(0), Some(second_ref)); + } + + #[test] + fn prefix_rejects_fork_above_closed_tip() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (genesis, weak) = genesis_parents(&committee, 0); + let first = candidate(&committee, 0, 1, genesis, weak, Vec::new(), 1).unwrap(); + let first_ref = first.reference(); + force_deliver(&mut model, first); + model.mark_data_available(first_ref).unwrap(); + + let mut unavailable_fork = first_ref; + unavailable_fork.digest = crate::types::BlockDigest::from([0x88; 32]); + let fork = candidate( + &committee, + 0, + 2, + unavailable_fork, + vec![ + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ], + Vec::new(), + 2, + ) + .unwrap(); + let fork_ref = fork.reference(); + force_deliver(&mut model, fork); + model.mark_data_available(fork_ref).unwrap(); + assert_eq!(model.prefix_tip(0), Some(first_ref)); + assert!(!model.lifecycle(&fork_ref).unwrap().prefix_closed); + } + + #[test] + fn delayed_data_availability_closes_the_whole_waiting_prefix() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (genesis, weak) = genesis_parents(&committee, 0); + let first = candidate(&committee, 0, 1, genesis, weak, Vec::new(), 1).unwrap(); + let first_ref = first.reference(); + force_deliver(&mut model, first); + model.mark_data_available(first_ref).unwrap(); + let second = candidate( + &committee, + 0, + 2, + first_ref, + vec![ + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ], + Vec::new(), + 2, + ) + .unwrap(); + let second_ref = second.reference(); + force_deliver(&mut model, second); + let third = candidate( + &committee, + 0, + 3, + second_ref, + vec![ + BlockReference::new_test(1, 2), + BlockReference::new_test(2, 2), + ], + Vec::new(), + 3, + ) + .unwrap(); + let third_ref = third.reference(); + force_deliver(&mut model, third); + model.mark_data_available(third_ref).unwrap(); + assert_eq!(model.prefix_tip(0), Some(first_ref)); + + let effects = model.mark_data_available(second_ref).unwrap(); + assert_eq!(model.prefix_tip(0), Some(third_ref)); + assert_eq!( + effects + .iter() + .filter(|effect| matches!(effect, ModelEffect::PrefixAdvanced { .. })) + .count(), + 2 + ); + } + + #[test] + fn equal_frontiers_produce_identical_ordered_deltas() { + let committee = committee(4); + let mut left = model(Arc::clone(&committee), 3); + let mut right = model(Arc::clone(&committee), 3); + let carriers: Vec<_> = committee + .authorities() + .map(|author| { + let (own_prev, weak) = genesis_parents(&committee, author); + candidate( + &committee, + author, + 1, + own_prev, + weak, + Vec::new(), + u64::from(author), + ) + .unwrap() + }) + .collect(); + for carrier in &carriers { + force_deliver(&mut left, carrier.clone()); + left.mark_data_available(carrier.reference()).unwrap(); + } + for carrier in carriers.iter().rev() { + force_deliver(&mut right, carrier.clone()); + right.mark_data_available(carrier.reference()).unwrap(); + } + let frontier: Vec<_> = carriers + .iter() + .map(|carrier| Some(carrier.reference())) + .collect(); + let left_delta = left.apply_frontier(&frontier).unwrap(); + let right_delta = right.apply_frontier(&frontier).unwrap(); + assert_eq!(left_delta, right_delta); + assert!(left_delta.windows(2).all(|pair| pair[0] < pair[1])); + assert!(left.apply_frontier(&frontier).unwrap().is_empty()); + } +} diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs new file mode 100644 index 00000000..86fc5274 --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -0,0 +1,1548 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Pure executable model of the certified Starfish projection. +//! +//! This module deliberately has no network, storage, pacemaker, or production +//! consensus integration. It models the boundary at which an RBC-delivered, +//! data-available carrier may contribute its optional consensus vertex, and it +//! evaluates the explicit vote/no-vote evidence committed by those vertices. + +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, + sync::Arc, +}; + +use crate::{ + committee::Committee, + types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, +}; + +use super::{ + CandidateCarrierV1, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, + RbcDagCommitteeId, RbcDagProjectionError, carrier_genesis_reference, +}; + +/// An indexed exact carrier-prefix frontier. `None` is the authority's virtual +/// genesis prefix; `Some` always identifies an exact carrier value. +pub type DeliveryFrontierV1 = Vec>; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct LeaderSlotV1 { + pub author: AuthorityIndex, + pub round: RoundNumber, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProjectionDecisionV1 { + DirectCommit { + leader: ConsensusVertexReference, + }, + DirectSkip { + slot: LeaderSlotV1, + }, + IndirectCommit { + leader: ConsensusVertexReference, + anchor: ConsensusVertexReference, + }, + IndirectSkip { + slot: LeaderSlotV1, + anchor: ConsensusVertexReference, + }, + Undecided { + slot: LeaderSlotV1, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CertifiedProjectionError { + CommitteeMismatch, + UnknownCarrier(BlockReference), + ConflictingDeliveredCarrierSlot { + existing: BlockReference, + conflicting: BlockReference, + }, + MissingConsensusVertex(BlockReference), + InvalidProjectionShape(RbcDagProjectionError), + CarrierNotDelivered(BlockReference), + CarrierDataUnavailable(BlockReference), + CarrierOutsideClosedPrefix(BlockReference), + MissingStrongParent(ConsensusVertexReference), + InvalidGenesisStrongParent(ConsensusVertexReference), + OwnFrontierDoesNotNamePreviousCarrier { + expected: Option, + actual: Option, + }, + FrontierNotClosed { + authority: AuthorityIndex, + reference: BlockReference, + }, + ParentFrontierFork { + authority: AuthorityIndex, + left: Option, + right: Option, + }, + FrontierDoesNotDominateParent { + authority: AuthorityIndex, + required: Option, + actual: Option, + }, + FrontierRegressesCommitted { + authority: AuthorityIndex, + committed: Option, + actual: Option, + }, + StakeOverflow, + InvalidLeaderSlot(LeaderSlotV1), + MultipleCertifiedLeaderValues(LeaderSlotV1), + ConflictingDirectDecision(LeaderSlotV1), + AnchorNotCommitted(ConsensusVertexReference), + AnchorTooEarly { + slot: LeaderSlotV1, + anchor: ConsensusVertexReference, + }, +} + +impl fmt::Display for CertifiedProjectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "certified projection error: {self:?}") + } +} + +impl Error for CertifiedProjectionError {} + +#[derive(Clone, Debug)] +struct CarrierState { + candidate: CandidateCarrierV1, + delivered: bool, + data_available: bool, +} + +#[derive(Clone, Debug)] +struct ProjectedVertex { + vertex: ConsensusVertexV1, + effective_frontier: DeliveryFrontierV1, +} + +/// Stateful, deterministic model of the clean certified projection. +/// +/// Staging and cleaning a carrier never projects its optional vertex +/// automatically. Callers explicitly invoke [`Self::try_project`] so tests can +/// observe malformed or unavailable optional metadata without changing carrier +/// admission state. +#[derive(Clone)] +pub struct CertifiedProjectionModel { + committee: Arc, + committee_id: RbcDagCommitteeId, + carriers: BTreeMap, + delivered_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, + closed_prefixes: Vec>, + vertices: BTreeMap, + consensus_slots: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, + committed_frontier: DeliveryFrontierV1, + committed_anchors: BTreeSet, +} + +impl CertifiedProjectionModel { + pub fn new(committee: Arc) -> Result { + let committee_id = RbcDagCommitteeId::derive(&committee) + .map_err(|_| CertifiedProjectionError::CommitteeMismatch)?; + let committee_size = committee.len(); + Ok(Self { + committee, + committee_id, + carriers: BTreeMap::new(), + delivered_slots: BTreeMap::new(), + closed_prefixes: vec![Vec::new(); committee_size], + vertices: BTreeMap::new(), + consensus_slots: BTreeMap::new(), + committed_frontier: vec![None; committee_size], + committed_anchors: BTreeSet::new(), + }) + } + + /// Retain a canonical carrier independently of optional-vertex validity. + pub fn stage_carrier( + &mut self, + candidate: CandidateCarrierV1, + ) -> Result<(), CertifiedProjectionError> { + if candidate.committee_id() != self.committee_id { + return Err(CertifiedProjectionError::CommitteeMismatch); + } + self.carriers + .entry(candidate.reference()) + .or_insert(CarrierState { + candidate, + delivered: false, + data_available: false, + }); + Ok(()) + } + + /// Record exact RBC delivery. A second delivered value in one physical + /// author/round slot is rejected rather than resolved by arrival order. + pub fn mark_delivered( + &mut self, + reference: BlockReference, + ) -> Result<(), CertifiedProjectionError> { + let state = self + .carriers + .get(&reference) + .ok_or(CertifiedProjectionError::UnknownCarrier(reference))?; + let slot = ( + state.candidate.header().author(), + state.candidate.header().carrier_round(), + ); + if let Some(existing) = self.delivered_slots.get(&slot) { + if *existing != reference { + return Err(CertifiedProjectionError::ConflictingDeliveredCarrierSlot { + existing: *existing, + conflicting: reference, + }); + } + } + self.delivered_slots.insert(slot, reference); + self.carriers + .get_mut(&reference) + .expect("carrier checked above") + .delivered = true; + self.advance_closed_prefix(slot.0); + Ok(()) + } + + pub fn mark_data_available( + &mut self, + reference: BlockReference, + ) -> Result<(), CertifiedProjectionError> { + let authority = self + .carriers + .get(&reference) + .ok_or(CertifiedProjectionError::UnknownCarrier(reference))? + .candidate + .header() + .author(); + self.carriers + .get_mut(&reference) + .expect("carrier checked above") + .data_available = true; + self.advance_closed_prefix(authority); + Ok(()) + } + + pub fn carrier_is_stored(&self, reference: BlockReference) -> bool { + self.carriers.contains_key(&reference) + } + + pub fn closed_tip(&self, authority: AuthorityIndex) -> Option { + self.closed_prefixes + .get(authority as usize) + .and_then(|prefix| prefix.last()) + .copied() + } + + pub fn is_projected(&self, reference: ConsensusVertexReference) -> bool { + self.vertices.contains_key(&reference) + } + + pub fn slot_values( + &self, + author: AuthorityIndex, + round: RoundNumber, + ) -> Vec { + self.consensus_slots + .get(&(author, round)) + .map(|values| values.iter().copied().collect()) + .unwrap_or_default() + } + + pub fn effective_frontier( + &self, + reference: ConsensusVertexReference, + ) -> Option<&[Option]> { + self.vertices + .get(&reference) + .map(|projected| projected.effective_frontier.as_slice()) + } + + pub fn leader_choice(&self, reference: ConsensusVertexReference) -> Option { + self.vertices + .get(&reference) + .map(|projected| projected.vertex.leader_choice()) + } + + /// Project one optional consensus vertex if every stateful eligibility + /// condition holds. Failure leaves the enclosing carrier untouched. + pub fn try_project( + &mut self, + carrier_reference: BlockReference, + ) -> Result { + let state = self + .carriers + .get(&carrier_reference) + .ok_or(CertifiedProjectionError::UnknownCarrier(carrier_reference))?; + let vertex = state.candidate.header().consensus_vertex().cloned().ok_or( + CertifiedProjectionError::MissingConsensusVertex(carrier_reference), + )?; + let vertex_reference = + ConsensusVertexReference::new(carrier_reference, vertex.consensus_round()); + if self.vertices.contains_key(&vertex_reference) { + return Ok(vertex_reference); + } + + state + .candidate + .validate_consensus_vertex(&self.committee) + .map_err(CertifiedProjectionError::InvalidProjectionShape)?; + if !state.delivered { + return Err(CertifiedProjectionError::CarrierNotDelivered( + carrier_reference, + )); + } + if !state.data_available { + return Err(CertifiedProjectionError::CarrierDataUnavailable( + carrier_reference, + )); + } + if !self.is_on_closed_prefix(carrier_reference) { + return Err(CertifiedProjectionError::CarrierOutsideClosedPrefix( + carrier_reference, + )); + } + + let author = carrier_reference.authority; + let own_previous = state.candidate.header().own_prev(); + let expected_author_frontier = (own_previous.round != 0).then_some(own_previous); + let actual_author_frontier = vertex.delivery_frontier()[author as usize]; + if actual_author_frontier != expected_author_frontier { + return Err( + CertifiedProjectionError::OwnFrontierDoesNotNamePreviousCarrier { + expected: expected_author_frontier, + actual: actual_author_frontier, + }, + ); + } + + self.ensure_frontier_closed(vertex.delivery_frontier())?; + + let mut parent_frontiers = Vec::with_capacity(vertex.strong_parents().len()); + for parent in vertex.strong_parents() { + if parent.consensus_round() == 0 { + if parent.carrier() != carrier_genesis_reference(parent.author()) { + return Err(CertifiedProjectionError::InvalidGenesisStrongParent( + *parent, + )); + } + parent_frontiers.push(vec![None; self.committee.len()]); + continue; + } + let projected = self + .vertices + .get(parent) + .ok_or(CertifiedProjectionError::MissingStrongParent(*parent))?; + parent_frontiers.push(projected.effective_frontier.clone()); + } + let parent_join = self.join_frontiers(&parent_frontiers)?; + self.ensure_dominates_parent(vertex.delivery_frontier(), &parent_join)?; + + let mut effective_frontier = vertex.delivery_frontier().to_vec(); + effective_frontier[author as usize] = Some(carrier_reference); + self.vertices.insert( + vertex_reference, + ProjectedVertex { + vertex, + effective_frontier, + }, + ); + self.consensus_slots + .entry((author, vertex_reference.consensus_round())) + .or_default() + .insert(vertex_reference); + Ok(vertex_reference) + } + + /// The scheduled leader slot for a consensus round. + pub fn leader_slot(&self, round: RoundNumber) -> LeaderSlotV1 { + LeaderSlotV1 { + author: self.committee.elect_leader(round), + round, + } + } + + /// Stake of distinct projected voter authors that explicitly vote for this + /// exact leader value. Equivocations by one author count once. + pub fn vote_stake( + &self, + leader: ConsensusVertexReference, + ) -> Result { + let slot = LeaderSlotV1 { + author: leader.author(), + round: leader.consensus_round(), + }; + self.validate_leader_slot(slot)?; + self.stake_of_authors(self.voter_authors( + slot, + |choice| matches!(choice, LeaderChoiceV1::Vote { leader: voted } if voted == leader), + )) + } + + /// Evaluate the explicit direct Starfish patterns over clean projected + /// vertices only. + pub fn direct_decision( + &self, + slot: LeaderSlotV1, + ) -> Result { + self.validate_leader_slot(slot)?; + let candidates = self.slot_values(slot.author, slot.round); + let mut committed = Vec::new(); + for candidate in &candidates { + let certifier_authors = self.certifier_authors(*candidate); + if self.stake_of_authors(certifier_authors)? >= self.committee.quorum_threshold() { + committed.push(*candidate); + } + } + if committed.len() > 1 { + return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues( + slot, + )); + } + + let all_choice_authors = self.voter_authors(slot, |_| true); + let enough_choices = + self.stake_of_authors(all_choice_authors)? >= self.committee.quorum_threshold(); + let skip = enough_choices + && candidates.iter().all(|candidate| { + self.stake_of_authors(self.voter_authors(slot, |choice| match choice { + LeaderChoiceV1::Vote { leader } => leader != *candidate, + LeaderChoiceV1::NoVote { .. } => true, + })) + .is_ok_and(|stake| stake >= self.committee.quorum_threshold()) + }); + + match (committed.pop(), skip) { + (Some(_), true) => Err(CertifiedProjectionError::ConflictingDirectDecision(slot)), + (Some(leader), false) => Ok(ProjectionDecisionV1::DirectCommit { leader }), + (None, true) => Ok(ProjectionDecisionV1::DirectSkip { slot }), + (None, false) => Ok(ProjectionDecisionV1::Undecided { slot }), + } + } + + /// Record an externally selected committed anchor while enforcing exact + /// componentwise frontier monotonicity. The runtime committer remains out + /// of scope for this model. + pub fn record_committed_anchor( + &mut self, + anchor: ConsensusVertexReference, + ) -> Result<(), CertifiedProjectionError> { + let projected = self + .vertices + .get(&anchor) + .ok_or(CertifiedProjectionError::MissingStrongParent(anchor))?; + let frontier = projected.effective_frontier.clone(); + self.ensure_dominates_committed(&frontier)?; + self.committed_frontier = frontier; + self.committed_anchors.insert(anchor); + Ok(()) + } + + /// Decide an older leader from a later committed anchor. A reachable + /// certifying-round vertex with a QC yields commit; absence yields skip. + pub fn indirect_decision( + &self, + slot: LeaderSlotV1, + anchor: ConsensusVertexReference, + ) -> Result { + self.validate_leader_slot(slot)?; + if !self.committed_anchors.contains(&anchor) { + return Err(CertifiedProjectionError::AnchorNotCommitted(anchor)); + } + let minimum_anchor_round = slot.round.saturating_add(3); + if anchor.consensus_round() < minimum_anchor_round { + return Err(CertifiedProjectionError::AnchorTooEarly { slot, anchor }); + } + let certifying_round = slot.round.saturating_add(2); + let reachable = self.reachable_at_round(anchor, certifying_round); + let mut certified = Vec::new(); + for candidate in self.slot_values(slot.author, slot.round) { + if reachable + .iter() + .any(|certifier| self.is_certificate(*certifier, candidate)) + { + certified.push(candidate); + } + } + if certified.len() > 1 { + return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues( + slot, + )); + } + Ok(match certified.pop() { + Some(leader) => ProjectionDecisionV1::IndirectCommit { leader, anchor }, + None => ProjectionDecisionV1::IndirectSkip { slot, anchor }, + }) + } + + fn advance_closed_prefix(&mut self, authority: AuthorityIndex) { + let index = authority as usize; + loop { + let next_round = self.closed_prefixes[index].len() as RoundNumber + 1; + let Some(reference) = self.delivered_slots.get(&(authority, next_round)).copied() + else { + break; + }; + let Some(state) = self.carriers.get(&reference) else { + break; + }; + if !state.delivered || !state.data_available { + break; + } + let expected_previous = self.closed_prefixes[index] + .last() + .copied() + .unwrap_or_else(|| carrier_genesis_reference(authority)); + if state.candidate.header().own_prev() != expected_previous { + break; + } + self.closed_prefixes[index].push(reference); + } + } + + fn is_on_closed_prefix(&self, reference: BlockReference) -> bool { + self.closed_tip(reference.authority) + .is_some_and(|tip| self.is_exact_extension(Some(reference), Some(tip))) + } + + fn ensure_frontier_closed( + &self, + frontier: &[Option], + ) -> Result<(), CertifiedProjectionError> { + for (index, entry) in frontier.iter().copied().enumerate() { + let Some(reference) = entry else { + continue; + }; + let authority = index as AuthorityIndex; + let closed_tip = self.closed_tip(authority); + if !self.is_exact_extension(Some(reference), closed_tip) { + return Err(CertifiedProjectionError::FrontierNotClosed { + authority, + reference, + }); + } + } + Ok(()) + } + + fn join_frontiers( + &self, + frontiers: &[DeliveryFrontierV1], + ) -> Result { + let mut joined = vec![None; self.committee.len()]; + for frontier in frontiers { + for (index, right) in frontier.iter().copied().enumerate() { + let left = joined[index]; + if self.is_exact_extension(left, right) { + joined[index] = right; + } else if !self.is_exact_extension(right, left) { + return Err(CertifiedProjectionError::ParentFrontierFork { + authority: index as AuthorityIndex, + left, + right, + }); + } + } + } + Ok(joined) + } + + fn ensure_dominates_parent( + &self, + frontier: &[Option], + required: &[Option], + ) -> Result<(), CertifiedProjectionError> { + for (index, (required, actual)) in required + .iter() + .copied() + .zip(frontier.iter().copied()) + .enumerate() + { + if !self.is_exact_extension(required, actual) { + return Err(CertifiedProjectionError::FrontierDoesNotDominateParent { + authority: index as AuthorityIndex, + required, + actual, + }); + } + } + Ok(()) + } + + fn ensure_dominates_committed( + &self, + frontier: &[Option], + ) -> Result<(), CertifiedProjectionError> { + for (index, (committed, actual)) in self + .committed_frontier + .iter() + .copied() + .zip(frontier.iter().copied()) + .enumerate() + { + if !self.is_exact_extension(committed, actual) { + return Err(CertifiedProjectionError::FrontierRegressesCommitted { + authority: index as AuthorityIndex, + committed, + actual, + }); + } + } + Ok(()) + } + + /// True iff `descendant` is the same exact prefix tip as `base`, or an + /// exact self-chain extension whose intermediate carrier headers are known. + fn is_exact_extension( + &self, + base: Option, + descendant: Option, + ) -> bool { + let Some(mut cursor) = descendant else { + return base.is_none(); + }; + let authority = cursor.authority; + if base.is_some_and(|base| base.authority != authority) { + return false; + } + let base_round = base.map_or(0, |reference| reference.round); + if cursor.round < base_round { + return false; + } + while cursor.round > base_round { + let Some(state) = self.carriers.get(&cursor) else { + return false; + }; + if state.candidate.header().author() != authority + || state.candidate.reference() != cursor + { + return false; + } + cursor = state.candidate.header().own_prev(); + } + match base { + Some(reference) => cursor == reference, + None => cursor == carrier_genesis_reference(authority), + } + } + + fn validate_leader_slot(&self, slot: LeaderSlotV1) -> Result<(), CertifiedProjectionError> { + if slot.round == 0 || self.committee.elect_leader(slot.round) != slot.author { + return Err(CertifiedProjectionError::InvalidLeaderSlot(slot)); + } + Ok(()) + } + + fn vertices_at_round( + &self, + round: RoundNumber, + ) -> impl Iterator { + self.vertices + .iter() + .filter(move |(reference, _)| reference.consensus_round() == round) + .map(|(reference, projected)| (*reference, projected)) + } + + fn voter_authors( + &self, + slot: LeaderSlotV1, + predicate: impl Fn(LeaderChoiceV1) -> bool, + ) -> BTreeSet { + self.vertices_at_round(slot.round.saturating_add(1)) + .filter_map(|(reference, projected)| { + let choice = projected.vertex.leader_choice(); + let belongs_to_slot = match choice { + LeaderChoiceV1::Vote { leader } => { + leader.author() == slot.author && leader.consensus_round() == slot.round + } + LeaderChoiceV1::NoVote { + leader_author, + leader_round, + } => leader_author == slot.author && leader_round == slot.round, + }; + (belongs_to_slot && predicate(choice)).then_some(reference.author()) + }) + .collect() + } + + fn certifier_authors(&self, leader: ConsensusVertexReference) -> BTreeSet { + self.vertices_at_round(leader.consensus_round().saturating_add(2)) + .filter_map(|(reference, _)| { + self.is_certificate(reference, leader) + .then_some(reference.author()) + }) + .collect() + } + + fn is_certificate( + &self, + certifier: ConsensusVertexReference, + leader: ConsensusVertexReference, + ) -> bool { + let Some(projected) = self.vertices.get(&certifier) else { + return false; + }; + let voter_authors: BTreeSet<_> = projected + .vertex + .strong_parents() + .iter() + .filter_map(|parent| { + self.vertices.get(parent).and_then(|voter| { + matches!( + voter.vertex.leader_choice(), + LeaderChoiceV1::Vote { leader: voted } if voted == leader + ) + .then_some(parent.author()) + }) + }) + .collect(); + self.stake_of_authors(voter_authors) + .is_ok_and(|stake| stake >= self.committee.quorum_threshold()) + } + + fn stake_of_authors( + &self, + authors: BTreeSet, + ) -> Result { + authors.into_iter().try_fold(0u64, |stake, author| { + let author_stake = self + .committee + .get_stake(author) + .ok_or(CertifiedProjectionError::StakeOverflow)?; + stake + .checked_add(author_stake) + .ok_or(CertifiedProjectionError::StakeOverflow) + }) + } + + fn reachable_at_round( + &self, + anchor: ConsensusVertexReference, + target_round: RoundNumber, + ) -> BTreeSet { + let mut result = BTreeSet::new(); + let mut pending = vec![anchor]; + let mut seen = BTreeSet::new(); + while let Some(reference) = pending.pop() { + if !seen.insert(reference) || reference.consensus_round() < target_round { + continue; + } + if reference.consensus_round() == target_round { + result.insert(reference); + continue; + } + if let Some(projected) = self.vertices.get(&reference) { + pending.extend(projected.vertex.strong_parents().iter().copied()); + } + } + result + } + + #[cfg(test)] + fn inject_projected_for_test( + &mut self, + reference: ConsensusVertexReference, + strong_parents: Vec, + leader_choice: LeaderChoiceV1, + ) { + let vertex = ConsensusVertexV1::new( + reference.consensus_round(), + strong_parents, + vec![None; self.committee.len()], + leader_choice, + ); + self.vertices.insert( + reference, + ProjectedVertex { + vertex, + effective_frontier: vec![None; self.committee.len()], + }, + ); + self.consensus_slots + .entry((reference.author(), reference.consensus_round())) + .or_default() + .insert(reference); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + crypto::TransactionsCommitment, + starfish_rbc_dag::{CarrierHeaderV1Args, LeaderChoiceV1}, + types::BlockDigest, + }; + + fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { + BlockReference { + authority, + round, + digest: BlockDigest::from([marker; 32]), + } + } + + fn consensus_reference( + authority: AuthorityIndex, + round: RoundNumber, + marker: u8, + ) -> ConsensusVertexReference { + ConsensusVertexReference::new(reference(authority, round + 20, marker), round) + } + + fn previous_carriers(committee: &Committee, round: RoundNumber) -> Vec { + committee + .authorities() + .map(|authority| { + if round == 0 { + carrier_genesis_reference(authority) + } else { + reference(authority, round, authority as u8 + 0x80) + } + }) + .collect() + } + + fn candidate( + committee: &Committee, + author: AuthorityIndex, + carrier_round: RoundNumber, + previous: &[BlockReference], + vertex: Option, + marker: u8, + ) -> CandidateCarrierV1 { + let weak_parents = previous + .iter() + .copied() + .filter(|parent| parent.authority != author) + .collect(); + CandidateCarrierV1::try_new( + CarrierHeaderV1Args { + author, + carrier_round, + own_prev: previous[author as usize], + weak_parents, + transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: vertex, + creation_time_ns: marker as u64, + }, + committee, + ) + .unwrap() + } + + fn clean( + model: &mut CertifiedProjectionModel, + candidate: CandidateCarrierV1, + ) -> BlockReference { + let reference = candidate.reference(); + model.stage_carrier(candidate).unwrap(); + model.mark_delivered(reference).unwrap(); + model.mark_data_available(reference).unwrap(); + reference + } + + fn first_consensus_round( + model: &mut CertifiedProjectionModel, + ) -> (Vec, Vec) { + let previous = previous_carriers(&model.committee, 0); + let strong_parents: Vec<_> = model + .committee + .authorities() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + let leader = strong_parents[0]; + let mut carriers = Vec::new(); + let mut vertices = Vec::new(); + let authors: Vec<_> = model.committee.authorities().collect(); + for author in authors { + let vertex = ConsensusVertexV1::new( + 1, + strong_parents.clone(), + vec![None; model.committee.len()], + LeaderChoiceV1::Vote { leader }, + ); + let carrier = candidate( + &model.committee, + author, + 1, + &previous, + Some(vertex), + 0x10 + author as u8, + ); + let carrier_reference = clean(model, carrier); + let vertex_reference = model.try_project(carrier_reference).unwrap(); + carriers.push(carrier_reference); + vertices.push(vertex_reference); + } + (carriers, vertices) + } + + fn second_round_candidate( + model: &CertifiedProjectionModel, + author: AuthorityIndex, + previous: &[BlockReference], + parents: Vec, + frontier: DeliveryFrontierV1, + marker: u8, + ) -> CandidateCarrierV1 { + let leader = parents + .iter() + .find(|parent| parent.author() == model.committee.elect_leader(1)) + .copied() + .unwrap(); + candidate( + &model.committee, + author, + 2, + previous, + Some(ConsensusVertexV1::new( + 2, + parents, + frontier, + LeaderChoiceV1::Vote { leader }, + )), + marker, + ) + } + + #[test] + fn effective_frontier_includes_every_enclosing_strong_parent() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let (carriers, parents) = first_consensus_round(&mut model); + + let child = second_round_candidate( + &model, + 0, + &carriers, + parents, + carriers.iter().copied().map(Some).collect(), + 0x30, + ); + let child_carrier = clean(&mut model, child); + let child_vertex = model.try_project(child_carrier).unwrap(); + let effective = model.effective_frontier(child_vertex).unwrap(); + assert_eq!(effective[0], Some(child_carrier)); + let expected: Vec<_> = carriers[1..].iter().copied().map(Some).collect(); + assert_eq!(&effective[1..], expected.as_slice()); + } + + #[test] + fn omission_and_same_round_fork_do_not_pass_frontier_checks() { + let committee = Committee::new_test(vec![1; 4]); + let mut omission_model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let (carriers, parents) = first_consensus_round(&mut omission_model); + let mut omitted = carriers.iter().copied().map(Some).collect::>(); + omitted[2] = None; + let child = second_round_candidate(&omission_model, 0, &carriers, parents, omitted, 0x31); + let child_reference = clean(&mut omission_model, child); + assert!(matches!( + omission_model.try_project(child_reference), + Err(CertifiedProjectionError::FrontierDoesNotDominateParent { authority: 2, .. }) + )); + assert!(omission_model.carrier_is_stored(child_reference)); + + let mut fork_model = CertifiedProjectionModel::new(committee).unwrap(); + let (carriers, parents) = first_consensus_round(&mut fork_model); + let mut forked = carriers.iter().copied().map(Some).collect::>(); + forked[2] = Some(reference(2, 1, 0xEE)); + let child = second_round_candidate(&fork_model, 0, &carriers, parents, forked, 0x32); + let child_reference = clean(&mut fork_model, child); + assert!(matches!( + fork_model.try_project(child_reference), + Err(CertifiedProjectionError::FrontierNotClosed { authority: 2, .. }) + )); + } + + #[test] + fn exact_strong_parent_lookup_and_parent_fork_are_enforced() { + let committee = Committee::new_test(vec![1; 4]); + let mut exact_model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let (carriers, mut parents) = first_consensus_round(&mut exact_model); + parents[2] = ConsensusVertexReference::new(reference(2, 1, 0xEF), 1); + let child = second_round_candidate( + &exact_model, + 0, + &carriers, + parents, + carriers.iter().copied().map(Some).collect(), + 0x33, + ); + let child_reference = clean(&mut exact_model, child); + assert!(matches!( + exact_model.try_project(child_reference), + Err(CertifiedProjectionError::MissingStrongParent(parent)) + if parent.author() == 2 + )); + + let mut fork_model = CertifiedProjectionModel::new(committee).unwrap(); + let genesis = previous_carriers(&fork_model.committee, 0); + let base = candidate(&fork_model.committee, 0, 1, &genesis, None, 0x34); + let base_reference = clean(&mut fork_model, base); + let parent_references: Vec<_> = (0..3) + .map(|authority| consensus_reference(authority, 1, 0x40 + authority as u8)) + .collect(); + for parent in &parent_references { + fork_model.inject_projected_for_test( + *parent, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }, + ); + } + fork_model + .vertices + .get_mut(&parent_references[0]) + .unwrap() + .effective_frontier[3] = Some(reference(3, 1, 0xA1)); + fork_model + .vertices + .get_mut(&parent_references[1]) + .unwrap() + .effective_frontier[3] = Some(reference(3, 1, 0xA2)); + let previous = vec![ + base_reference, + reference(1, 1, 0xB1), + reference(2, 1, 0xB2), + reference(3, 1, 0xB3), + ]; + let child = second_round_candidate( + &fork_model, + 0, + &previous, + parent_references, + vec![Some(base_reference), None, None, None], + 0x35, + ); + let child_reference = clean(&mut fork_model, child); + assert!(matches!( + fork_model.try_project(child_reference), + Err(CertifiedProjectionError::ParentFrontierFork { authority: 3, .. }) + )); + } + + #[test] + fn late_vertex_remains_visible_but_cannot_become_a_regressing_anchor() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (carriers, parents) = first_consensus_round(&mut model); + let anchor = second_round_candidate( + &model, + 0, + &carriers, + parents.clone(), + carriers.iter().copied().map(Some).collect(), + 0x36, + ); + let anchor_carrier = clean(&mut model, anchor); + let anchor_vertex = model.try_project(anchor_carrier).unwrap(); + model.record_committed_anchor(anchor_vertex).unwrap(); + + let regressing = second_round_candidate( + &model, + 2, + &carriers, + parents, + carriers.iter().copied().map(Some).collect(), + 0x37, + ); + let regressing_carrier = clean(&mut model, regressing); + let regressing_vertex = model.try_project(regressing_carrier).unwrap(); + assert!(model.is_projected(regressing_vertex)); + assert!(matches!( + model.record_committed_anchor(regressing_vertex), + Err(CertifiedProjectionError::FrontierRegressesCommitted { authority: 0, .. }) + )); + } + + #[test] + fn conflicting_consensus_values_in_one_slot_remain_visible() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (carriers, _) = first_consensus_round(&mut model); + let genesis_parents: Vec<_> = model + .committee + .authorities() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + let conflicting = candidate( + &model.committee, + 0, + 2, + &carriers, + Some(ConsensusVertexV1::new( + 1, + genesis_parents.clone(), + carriers.iter().copied().map(Some).collect(), + LeaderChoiceV1::Vote { + leader: genesis_parents[0], + }, + )), + 0x38, + ); + let conflicting_carrier = clean(&mut model, conflicting); + model.try_project(conflicting_carrier).unwrap(); + + let values = model.slot_values(0, 1); + assert_eq!(values.len(), 2); + assert_ne!(values[0], values[1]); + } + + #[test] + fn objective_vote_and_no_vote_shapes_are_both_projectable() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let previous = previous_carriers(&committee, 0); + let genesis: Vec<_> = committee + .authorities() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + let vote = candidate( + &committee, + 1, + 1, + &previous, + Some(ConsensusVertexV1::new( + 1, + vec![genesis[0], genesis[1], genesis[2]], + vec![None; 4], + LeaderChoiceV1::Vote { leader: genesis[0] }, + )), + 0x39, + ); + let vote_carrier = clean(&mut model, vote); + let vote_reference = model.try_project(vote_carrier).unwrap(); + let no_vote = candidate( + &committee, + 3, + 1, + &previous, + Some(ConsensusVertexV1::new( + 1, + vec![genesis[1], genesis[2], genesis[3]], + vec![None; 4], + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }, + )), + 0x3A, + ); + let no_vote_carrier = clean(&mut model, no_vote); + let no_vote_reference = model.try_project(no_vote_carrier).unwrap(); + + assert_eq!( + model.leader_choice(vote_reference), + Some(LeaderChoiceV1::Vote { leader: genesis[0] }) + ); + assert!(matches!( + model.leader_choice(no_vote_reference), + Some(LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0 + }) + )); + } + + #[test] + fn dirty_or_malformed_optional_vertex_never_enters_projection() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let previous = previous_carriers(&committee, 0); + let genesis: Vec<_> = committee + .authorities() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + let dirty = candidate( + &committee, + 0, + 1, + &previous, + Some(ConsensusVertexV1::new( + 1, + vec![genesis[0], genesis[1], genesis[2]], + vec![None; 4], + LeaderChoiceV1::Vote { leader: genesis[0] }, + )), + 0x3B, + ); + let dirty_carrier = dirty.reference(); + model.stage_carrier(dirty).unwrap(); + assert_eq!( + model.try_project(dirty_carrier), + Err(CertifiedProjectionError::CarrierNotDelivered(dirty_carrier)) + ); + assert!(model.carrier_is_stored(dirty_carrier)); + assert!(model.slot_values(0, 1).is_empty()); + + let malformed = candidate( + &committee, + 1, + 1, + &previous, + Some(ConsensusVertexV1::new( + 1, + vec![genesis[1]], + vec![None; 4], + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }, + )), + 0x3C, + ); + let malformed_carrier = clean(&mut model, malformed); + assert!(matches!( + model.try_project(malformed_carrier), + Err(CertifiedProjectionError::InvalidProjectionShape( + RbcDagProjectionError::InvalidStrongParentThreshold + )) + )); + assert!(model.carrier_is_stored(malformed_carrier)); + assert!(model.slot_values(1, 1).is_empty()); + } + + #[test] + fn missing_weak_parent_bodies_do_not_block_consensus_projection() { + let committee = Committee::new_test(vec![1; 7]); + let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let genesis = previous_carriers(&committee, 0); + let base = candidate(&committee, 0, 1, &genesis, None, 0x3D); + let base_reference = clean(&mut model, base); + + let mut previous = previous_carriers(&committee, 1); + previous[0] = base_reference; + let missing: Vec<_> = previous[1..=2].to_vec(); + let strong_parents: Vec<_> = committee + .authorities() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + let leader = strong_parents + .iter() + .find(|parent| parent.author() == committee.elect_leader(0)) + .copied() + .unwrap(); + let mut frontier = vec![None; committee.len()]; + frontier[0] = Some(base_reference); + let outer = candidate( + &committee, + 0, + 2, + &previous, + Some(ConsensusVertexV1::new( + 1, + strong_parents, + frontier, + LeaderChoiceV1::Vote { leader }, + )), + 0x3E, + ); + let outer_reference = clean(&mut model, outer); + + assert!( + missing + .iter() + .all(|reference| !model.carrier_is_stored(*reference)) + ); + assert!(model.try_project(outer_reference).is_ok()); + } + + fn project_complete_round( + model: &mut CertifiedProjectionModel, + consensus_round: RoundNumber, + previous_carriers: &[BlockReference], + parent_sets: &[Vec], + choices: &[LeaderChoiceV1], + marker_base: u8, + ) -> (Vec, Vec) { + assert_eq!(parent_sets.len(), model.committee.len()); + assert_eq!(choices.len(), model.committee.len()); + let carrier_round = previous_carriers[0].round + 1; + let frontier: DeliveryFrontierV1 = previous_carriers.iter().copied().map(Some).collect(); + let mut carriers = Vec::with_capacity(model.committee.len()); + let mut vertices = Vec::with_capacity(model.committee.len()); + for author in 0..model.committee.len() as AuthorityIndex { + let carrier = candidate( + &model.committee, + author, + carrier_round, + previous_carriers, + Some(ConsensusVertexV1::new( + consensus_round, + parent_sets[author as usize].clone(), + frontier.clone(), + choices[author as usize], + )), + marker_base + author as u8, + ); + let carrier_reference = clean(model, carrier); + let vertex_reference = model.try_project(carrier_reference).unwrap(); + carriers.push(carrier_reference); + vertices.push(vertex_reference); + } + (carriers, vertices) + } + + #[test] + fn direct_commit_uses_clean_projected_voters_and_certifiers() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); + let slot = model.leader_slot(1); + let leader = round_one_vertices[slot.author as usize]; + + let voter_parents = vec![round_one_vertices.clone(); 4]; + let voter_choices = vec![LeaderChoiceV1::Vote { leader }; 4]; + let (round_two_carriers, round_two_vertices) = project_complete_round( + &mut model, + 2, + &round_one_carriers, + &voter_parents, + &voter_choices, + 0x50, + ); + assert_eq!(model.vote_stake(leader).unwrap(), 4); + + let round_two_leader = round_two_vertices[model.committee.elect_leader(2) as usize]; + let certifier_parents = vec![round_two_vertices; 4]; + let certifier_choices = vec![ + LeaderChoiceV1::Vote { + leader: round_two_leader, + }; + 4 + ]; + project_complete_round( + &mut model, + 3, + &round_two_carriers, + &certifier_parents, + &certifier_choices, + 0x60, + ); + + assert_eq!( + model.direct_decision(slot).unwrap(), + ProjectionDecisionV1::DirectCommit { leader } + ); + } + + #[test] + fn direct_skip_uses_clean_projected_explicit_negative_choices() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); + let slot = model.leader_slot(1); + let leader = round_one_vertices[slot.author as usize]; + let negative_parents = vec![ + round_one_vertices[0], + round_one_vertices[2], + round_one_vertices[3], + ]; + let voter_parents = vec![ + negative_parents.clone(), + vec![ + round_one_vertices[0], + round_one_vertices[1], + round_one_vertices[2], + ], + negative_parents.clone(), + negative_parents, + ]; + let no_vote = LeaderChoiceV1::NoVote { + leader_author: slot.author, + leader_round: slot.round, + }; + let voter_choices = vec![no_vote, LeaderChoiceV1::Vote { leader }, no_vote, no_vote]; + project_complete_round( + &mut model, + 2, + &round_one_carriers, + &voter_parents, + &voter_choices, + 0x70, + ); + + assert_eq!( + model.direct_decision(slot).unwrap(), + ProjectionDecisionV1::DirectSkip { slot } + ); + } + + fn indirect_graph( + include_certificate: bool, + ) -> ( + CertifiedProjectionModel, + LeaderSlotV1, + ConsensusVertexReference, + ConsensusVertexReference, + ) { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); + let slot = model.leader_slot(1); + let leader = round_one_vertices[slot.author as usize]; + + let voter_parent_sets = vec![ + vec![ + round_one_vertices[0], + round_one_vertices[1], + round_one_vertices[2], + ], + vec![ + round_one_vertices[0], + round_one_vertices[1], + round_one_vertices[2], + ], + vec![ + round_one_vertices[0], + round_one_vertices[1], + round_one_vertices[2], + ], + vec![ + round_one_vertices[0], + round_one_vertices[2], + round_one_vertices[3], + ], + ]; + let no_vote_round_one = LeaderChoiceV1::NoVote { + leader_author: slot.author, + leader_round: slot.round, + }; + let voter_choices = vec![ + LeaderChoiceV1::Vote { leader }, + LeaderChoiceV1::Vote { leader }, + LeaderChoiceV1::Vote { leader }, + no_vote_round_one, + ]; + let (round_two_carriers, round_two_vertices) = project_complete_round( + &mut model, + 2, + &round_one_carriers, + &voter_parent_sets, + &voter_choices, + 0x80, + ); + + let certifier_parent_sets = vec![ + vec![ + round_two_vertices[0], + round_two_vertices[1], + round_two_vertices[2], + ], + vec![ + round_two_vertices[0], + round_two_vertices[1], + round_two_vertices[3], + ], + vec![ + round_two_vertices[0], + round_two_vertices[2], + round_two_vertices[3], + ], + vec![ + round_two_vertices[1], + round_two_vertices[2], + round_two_vertices[3], + ], + ]; + let round_two_leader = round_two_vertices[model.committee.elect_leader(2) as usize]; + let no_vote_round_two = LeaderChoiceV1::NoVote { + leader_author: model.committee.elect_leader(2), + leader_round: 2, + }; + let certifier_choices = vec![ + LeaderChoiceV1::Vote { + leader: round_two_leader, + }, + no_vote_round_two, + LeaderChoiceV1::Vote { + leader: round_two_leader, + }, + LeaderChoiceV1::Vote { + leader: round_two_leader, + }, + ]; + let (round_three_carriers, round_three_vertices) = project_complete_round( + &mut model, + 3, + &round_two_carriers, + &certifier_parent_sets, + &certifier_choices, + 0x90, + ); + assert_eq!( + model.direct_decision(slot).unwrap(), + ProjectionDecisionV1::Undecided { slot } + ); + + let (anchor_author, anchor_parents, anchor_choice) = if include_certificate { + ( + 0, + round_three_vertices[..3].to_vec(), + LeaderChoiceV1::NoVote { + leader_author: model.committee.elect_leader(3), + leader_round: 3, + }, + ) + } else { + ( + 3, + round_three_vertices[1..].to_vec(), + LeaderChoiceV1::Vote { + leader: round_three_vertices[3], + }, + ) + }; + let anchor_carrier = candidate( + &model.committee, + anchor_author, + 4, + &round_three_carriers, + Some(ConsensusVertexV1::new( + 4, + anchor_parents, + round_three_carriers.iter().copied().map(Some).collect(), + anchor_choice, + )), + if include_certificate { 0xA0 } else { 0xA1 }, + ); + let anchor_carrier = clean(&mut model, anchor_carrier); + let anchor = model.try_project(anchor_carrier).unwrap(); + model.record_committed_anchor(anchor).unwrap(); + (model, slot, leader, anchor) + } + + #[test] + fn later_committed_anchor_drives_indirect_commit_or_skip() { + let (commit_model, slot, leader, commit_anchor) = indirect_graph(true); + assert_eq!( + commit_model.indirect_decision(slot, commit_anchor).unwrap(), + ProjectionDecisionV1::IndirectCommit { + leader, + anchor: commit_anchor, + } + ); + + let (skip_model, slot, _, skip_anchor) = indirect_graph(false); + assert_eq!( + skip_model.indirect_decision(slot, skip_anchor).unwrap(), + ProjectionDecisionV1::IndirectSkip { + slot, + anchor: skip_anchor, + } + ); + } +} diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index f9af20a9..d08bf7d8 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,6 +1,6 @@ # Starfish-RBC-DAG protocol design -Status: design milestone; no implementation or safety/liveness claim yet +Status: milestone-two reference codec and executable model; no runtime or safety/liveness claim The provisional CLI name for this protocol is `starfish-rbc-dag`. It is a new protocol, not a transport option or a version-two alias for `starfish-rbc`. @@ -33,10 +33,17 @@ remain selectable outer-authentication baselines; changing that selector does no embedded RBC or consensus rules. This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish -commit rules already exist, but their composition through two clocks, two logical projections, and -frontier-based payload ordering still requires an executable model, adversarial tests, and a proof. -Until those obligations are discharged, `starfish-rbc-dag` must be described as an experimental -prototype rather than a proven signature-free Starfish variant. +commit rules already exist. The isolated milestone-two implementation now provides a canonical +codec plus deterministic carrier/RBC, certified-projection, decision, and crash-journal models. +Those models are not runtime integration or a proof: the composition through two clocks, two +logical projections, and frontier-based payload ordering still requires shadow execution, +additional adversarial testing, and a safety/liveness argument. Until those obligations are +discharged, `starfish-rbc-dag` must be described as an experimental reference model rather than a +proven signature-free Starfish variant. + +The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified +Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not +payload reconstruction or the runtime transition from delivered acknowledgments to that input. Transaction bytes remain outside header RBC. The existing Reed-Solomon dissemination, acknowledgment, reconstruction, and transaction-commitment checks remain responsible for data @@ -97,8 +104,8 @@ strong parents and certified frontier constrain consensus. ## 4. Canonical objects -The milestone-two codec should implement the following logical types. Field widths, enum codes, -maximum lengths, and golden bytes are frozen by that milestone, before runtime integration. +The milestone-two codec implements the following logical types. Field widths, enum codes, maximum +lengths, and golden bytes are frozen before runtime integration. ```rust struct CarrierHeaderV1 { @@ -125,8 +132,7 @@ struct ConsensusVertexV1 { consensus_round: RoundNumber, strong_parents: Vec, delivery_frontier: Vec>, - // None only for the fixed genesis consensus round. - leader_choice: Option, + leader_choice: LeaderChoiceV1, } struct ConsensusVertexReference { @@ -150,18 +156,30 @@ For non-genesis carrier round `r`: - every weak parent has carrier round `r - 1` and a distinct non-local author; - the stake of `{ own_prev } union weak_parents` reaches `Q`; - phase targets have carrier rounds strictly below `r`; and -- every vector is canonically ordered and duplicate-free. +- weak parents are in strict authority order and duplicate-free; +- acknowledgments commit to the unique normalized sequence described below, and phase batches are + order-significant logs whose exact order is committed; +- a phase batch contains at most one statement for each `(phase, target author, target round)`; and +- strong-parent ordering, frontier indexing, and leader-choice validity are checked only when the + optional consensus vertex is projected, so an ineligible vertex cannot invalidate its carrier. + +The first encoded carrier has round one and names the fixed virtual round-zero carrier reference of +its author as `own_prev`; no round-zero carrier is sent on the wire. Consensus genesis is likewise +virtual. Every embedded consensus vertex has a positive consensus round and an explicit leader +choice. These conventions avoid making genesis a second, partially authenticated wire format. Weak references are syntax and pacing declarations, not availability assertions. Their target headers need not be present to authenticate, admit, process, or RBC-deliver the enclosing carrier. -Acknowledgments retain Starfish's logical order and compression on the wire. The content digest -commits to the expanded logical vector, while the stored compression must be canonical. An honest -author creates an acknowledgment only after the exact target is locally RBC-delivered and its -transaction data reconstructs to the committed root. The acknowledgment becomes usable as -data-availability evidence only after its enclosing carrier is also locally RBC-delivered; an -optimistically admitted Byzantine carrier cannot create inconsistent availability facts at -different validators. +Acknowledgments have one canonical logical order: first the unique maximal suffix shared with +`[own_prev] || weak_parents`, then all remaining acknowledgments in their original relative order. +The content digest commits to this expanded, suffix-first sequence, while the wire codec stores the +shared suffix as an intersection index and retains the order-significant extras. Non-canonical wire +aliases and duplicate acknowledgments are rejected. An honest author creates an acknowledgment +only after the exact target is locally RBC-delivered and its transaction data reconstructs to the +committed root. The acknowledgment becomes usable as data-availability evidence only after its +enclosing carrier is also locally RBC-delivered; an optimistically admitted Byzantine carrier +cannot create inconsistent availability facts at different validators. `delivery_frontier` has exactly one indexed entry per committee authority. `None` denotes that authority's fixed genesis/empty prefix. A `Some(reference)` entry must name the same authority as @@ -185,10 +203,42 @@ phase batch and optional consensus vertex. It excludes: - receipt peer, arrival time, and local admission state; and - recovery or transport metadata. -The byte grammar uses fixed field markers, fixed-width integers, and explicit vector lengths. It -does not add a `starfish:block-ref:v2` string to the block identity. A format-version field and the -unambiguous grammar distinguish this carrier layout; changing the layout requires a new version and -new golden vectors. +The byte grammar uses a one-byte format version, fixed field markers, big-endian fixed-width +integers, and explicit vector lengths. It does not add a `starfish:block-ref:v2` string to the block +identity. The format byte and unambiguous grammar distinguish this carrier layout; changing the +layout requires a new version and new golden vectors. The canonical identity codec is handwritten; +serde or bincode framing is never hashed. + +Milestone two freezes the version-one identity grammar as follows. `Ref` is +`author:u16 || carrier_round:u32 || digest:[u8;32]`; every integer is big-endian and every vector +count is `u16`. + +```text +00 01 +01 author:u16 +02 carrier_round:u32 +03 own_prev:Ref +04 weak_count:u16 weak:Ref[] +05 transactions_commitment:[u8;32] +06 acknowledgment_count:u16 expanded_acknowledgments:Ref[] +07 phase_count:u16 (phase:u8 target:Ref)[] // ECHO=0, READY=1 +08 consensus_present:u8 [ConsensusVertexV1] +09 creation_time_ns:u64 +``` + +The optional consensus encoding uses markers `01` through `04` for consensus round, strong +parents, delivery frontier, and leader choice. A strong reference is `Ref || consensus_round:u32`; +frontier entries use `0=None` and `1=Some(Ref)`; leader choices use `1=Vote` and `2=NoVote` (`0` is +reserved for virtual genesis and is rejected on the wire). The canonical transport codec replaces +the expanded acknowledgment field with `intersection_start:u16 || extra_count:u16 || extras`, where +the intersection is the unique maximal suffix of `[own_prev] || weak_parents`. Decoding expands and +recompresses this field and rejects aliases. To keep the two byte grammars self-describing, this +compressed transport form starts with `00 81`; only expanded identity content starts with `00 01`. + +Version one caps canonical carrier content at 4 MiB, weak and strong parents at the committee size, +the frontier at exactly the committee size when projected, and encoded phase batches at +`min(4n, 2048)`. The `4n` bound gives two times the expected `2n` steady-state phase arrival rate; +the scheduler still needs the fair-prefix and active-window rules described in Section 8.3. Consensus vertices are referenced by their exact enclosing `BlockReference` plus their declared `consensus_round`. Because there is at most one consensus vertex per carrier, that pair identifies @@ -407,12 +457,18 @@ authenticated holders eventually obtains the value after GST. ### 8.3 Batching and fairness -Phase batches are bounded. A deterministic fair queue must prevent Byzantine traffic for one slot +Phase batches are bounded. The encoded order is preserved and processed as an authenticated log; +two different orders intentionally identify different carriers. A deterministic fair queue must +prevent Byzantine traffic for one slot from starving honest ECHO/READY actions for other slots. In steady state, one authority can owe one ECHO and one READY for each of `n` previous-round carriers, so `2n` is the expected arrival rate and -not a safe capacity. The executable model initially uses an unbounded fair queue. A bounded runtime -must reserve strictly more than `2n` statements per carrier, plus an active-slot window, so delayed -work drains instead of remaining at permanent saturation. +not a safe capacity. The executable model retains an unbounded pending FIFO and drains the first +`4n` statements eligible for the carrier being built (capped by the version-one codec limit of +2,048 statements). A temporarily ineligible future-round statement remains in its stable queue +position but does not block older eligible work behind it. This exercises backlog, runahead, and +batching without pretending to solve adversarial fairness. A bounded runtime must use a fair +per-slot scheduler, reserve strictly more than `2n` statements per carrier, and enforce an +active-slot window so delayed work drains instead of remaining at permanent saturation. ## 9. Certified consensus vertices @@ -622,11 +678,20 @@ vector dissemination is a later optimization and requires redundant routes or di An authoritative implementation must persist proof-critical choices before exposing effects: -1. journal authenticated inbound provenance and its local ingress sequence; -2. persist local ECHO, READY, explicit no-vote, delivery, carrier-slot, and consensus-slot locks; -3. construct and persist the exact outbound carrier bytes, reference, and authentication sidecar; -4. only then send the carrier; and -5. after restart, replay the journal in recorded order and retransmit the identical carrier. +1. journal typed authenticated inbound provenance, exact bytes, and its local ingress sequence; +2. before fixing a local slot, construct and persist the typed candidate plus its exact canonical + carrier bytes and reference; +3. persist local ECHO, READY, explicit leader-choice, delivery, carrier-slot, and consensus-slot + locks that match that retained candidate (recovered content is likewise retained before READY); +4. persist the exact authentication sidecar and an outbound-exposure marker, and only then send the + carrier; and +5. after restart, replay the journal in recorded order and retransmit the identical carrier and + sidecar. + +Persisting a bare local reference before its canonical carrier bytes is not sufficient: a crash in +that gap would leave the slot fixed without the data needed to reconstruct the exact carrier. The +write-ahead model therefore makes content retention precede slot fixation and prevents exposure +until every lock encoded by that carrier is durable. Every persisted slot, candidate, lifecycle predicate, journal entry, and outbound-carrier key is namespaced by both `protocol_instance` and `committee_id`; storage from another run or committee @@ -718,7 +783,8 @@ minimum it must cover: - equal committed anchors producing byte-identical output deltas; - delayed data availability followed by eventual prefix inclusion; - crash points before and after each persisted lock and outbound-carrier write; and -- shadow replay matching the current direct RBC kernel's delivered references. +- once milestone three supplies the non-authoritative runtime path, shadow replay matching the + current direct RBC kernel's delivered references. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -752,11 +818,13 @@ Every milestone is committed separately. 1. **Protocol specification (this document):** lock the two clocks, lifecycle, full-vector sidecar, embedded Bracha transitions, certified prefix/frontier, commit/skip boundary, proof obligations, and experiment plan. No protocol code or CLI selector is added. -2. **Canonical codec and executable model:** add isolated carrier, phase, consensus, frontier, and - sidecar types; golden encodings; a pure in-memory state machine; and deterministic adversarial - simulations. No network or existing consensus path changes. +2. **Canonical codec and executable model (implemented):** isolated carrier, phase, consensus, + frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable + journal models; and deterministic adversarial simulations. No network or existing consensus path + changes. 3. **Persisted shadow carrier path:** build and store carriers alongside the current direct - `starfish-rbc` service, journal ingress and local locks, and compare embedded versus direct RBC + `starfish-rbc` service, cache the validated committee/domain identity rather than re-hashing all + public keys per carrier, journal ingress and local locks, and compare embedded versus direct RBC delivery. Direct RBC remains authoritative; shadow results never affect proposals or commits. 4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses @@ -777,8 +845,8 @@ Every milestone is committed separately. The following values are not safe to guess in the documentation milestone and must be resolved by the executable model or measured prototype: -- exact canonical field widths and maximum phase-batch size; -- maximum future-carrier buffer and payload runahead; +- production maximum future-carrier buffer and payload runahead (the executable model deliberately + uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); - the control-heartbeat rate under low load and backpressure; - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; From 445736ba7ba20176ed08667526113e47969510ee Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:04:02 +0200 Subject: [PATCH 30/62] Add persisted Starfish-RBC-DAG shadow runtime --- README.md | 27 +- crates/orchestrator/src/benchmark.rs | 48 +- crates/orchestrator/src/main.rs | 16 + crates/orchestrator/src/measurements.rs | 670 +++++ crates/orchestrator/src/orchestrator.rs | 140 +- crates/orchestrator/src/protocol/starfish.rs | 102 +- crates/starfish-core/src/config.rs | 12 + .../starfish-core/src/core_thread/spawned.rs | 11 +- crates/starfish-core/src/lib.rs | 2 + crates/starfish-core/src/metrics.rs | 300 ++ crates/starfish-core/src/net_sync.rs | 427 +++ crates/starfish-core/src/network.rs | 194 +- .../src/starfish_rbc_dag/journal.rs | 176 +- .../starfish-core/src/starfish_rbc_dag/mod.rs | 797 ++++- .../src/starfish_rbc_dag/model.rs | 795 ++++- .../src/starfish_rbc_dag/projection.rs | 28 +- .../src/starfish_rbc_dag/storage.rs | 1388 +++++++++ .../src/starfish_rbc_dag_shadow.rs | 2659 +++++++++++++++++ .../src/starfish_rbc_dag_shadow_service.rs | 2071 +++++++++++++ crates/starfish-core/src/syncer.rs | 28 +- crates/starfish-core/src/validator.rs | 142 + crates/starfish/src/main.rs | 88 +- docs/starfish-rbc-dag-protocol.md | 106 +- 23 files changed, 10066 insertions(+), 161 deletions(-) create mode 100644 crates/starfish-core/src/starfish_rbc_dag/storage.rs create mode 100644 crates/starfish-core/src/starfish_rbc_dag_shadow.rs create mode 100644 crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs diff --git a/README.md b/README.md index 3408e20c..7445b861 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,27 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is a codec-and-model-only follow-up that pipelines all-carrier RBC through an -optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate -logical projection. Its canonical types and deterministic executable models are implemented in -isolation, but there is no network/runtime path and no safety or liveness claim. Its provisional CLI -name is `starfish-rbc-dag`, but that selector is not implemented yet. The design, current boundary, -and proof obligations are documented in the [protocol design](docs/starfish-rbc-dag-protocol.md). +**Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG +while keeping certified Starfish consensus and ordering in a separate logical projection. Its +canonical types, deterministic models, crash journal, and an opt-in persisted network shadow are +implemented. Run the shadow with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; direct +Starfish-RBC remains solely authoritative and shadow failures or results cannot affect proposals, +commits, or output as protocol state. Shadow traffic still shares the validator's network socket and +bandwidth, so it can perturb timing, and it must be enabled only on a homogeneous new-binary +committee; there is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` +selector is not implemented yet. The shadow uses per-transition fsync and a clone-based reference +reducer, so it is a correctness instrument, not a fair performance baseline, and carries no safety +or liveness claim. Its WAL can reopen the shadow actor against matching recovered direct headers, +but this is not full validator crash recovery: authoritative direct Starfish-RBC phase and delivery +locks are not durable yet, so that baseline remains fail-stop across process restart. The design and +proof obligations are documented in the +[protocol design](docs/starfish-rbc-dag-protocol.md). +For any shadow comparison, +`starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded +observational path was disabled or shed work and the comparison must be discarded. Healthy live +production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded +unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between +the cumulative direct and shadow delivery counters. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index cec244f1..dc024546 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -99,6 +99,32 @@ pub struct BenchmarkRunSummary { pub ready_nodes_at_boot: usize, #[serde(default)] pub metrics_contributors: usize, + #[serde(default)] + pub shadow_comparison_enabled: bool, + #[serde(default)] + pub shadow_comparison_valid: bool, + #[serde(default)] + pub shadow_comparison_valid_nodes: usize, + #[serde(default)] + pub shadow_direct_deliveries: usize, + #[serde(default)] + pub shadow_deliveries: usize, + #[serde(default)] + pub shadow_delivery_matches: usize, + #[serde(default)] + pub shadow_delivery_mismatches: usize, + #[serde(default)] + pub shadow_delivery_ambiguous: usize, + #[serde(default)] + pub shadow_wal_durable_records: usize, + #[serde(default)] + pub shadow_pending_recovery: usize, + #[serde(default)] + pub shadow_unpaired_direct: usize, + #[serde(default)] + pub shadow_unpaired_shadow: usize, + #[serde(default)] + pub shadow_unpaired_max_round_lag: usize, } impl BenchmarkRunSummary { @@ -117,7 +143,14 @@ impl BenchmarkRunSummary { db_size_per_round_p25_bytes,db_size_per_round_p50_bytes,\ db_size_per_round_p75_bytes,\ block_sync_requests_sent_per_round_avg,block_header_size_avg_bytes,\ - ready_nodes_at_boot,metrics_contributors" + ready_nodes_at_boot,metrics_contributors,\ + shadow_comparison_enabled,shadow_comparison_valid,\ + shadow_comparison_valid_nodes,shadow_direct_deliveries,shadow_deliveries,\ + shadow_delivery_matches,\ + shadow_delivery_mismatches,shadow_delivery_ambiguous,\ + shadow_wal_durable_records,shadow_pending_recovery,\ + shadow_unpaired_direct,shadow_unpaired_shadow,\ + shadow_unpaired_max_round_lag" } pub fn csv_record(&self) -> String { @@ -151,6 +184,19 @@ impl BenchmarkRunSummary { format!("{:.3}", self.block_header_size_avg_bytes), self.ready_nodes_at_boot.to_string(), self.metrics_contributors.to_string(), + self.shadow_comparison_enabled.to_string(), + self.shadow_comparison_valid.to_string(), + self.shadow_comparison_valid_nodes.to_string(), + self.shadow_direct_deliveries.to_string(), + self.shadow_deliveries.to_string(), + self.shadow_delivery_matches.to_string(), + self.shadow_delivery_mismatches.to_string(), + self.shadow_delivery_ambiguous.to_string(), + self.shadow_wal_durable_records.to_string(), + self.shadow_pending_recovery.to_string(), + self.shadow_unpaired_direct.to_string(), + self.shadow_unpaired_shadow.to_string(), + self.shadow_unpaired_max_round_lag.to_string(), ] .join(",") } diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 1e39dcc9..eff22b62 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -63,6 +63,10 @@ pub struct Opts { #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, + /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative shadow. + #[clap(long, global = true)] + starfish_rbc_dag_shadow: bool, + /// The type of operation to run. #[clap(subcommand)] operation: Operation, @@ -852,6 +856,7 @@ fn load_benchmark_configs( compress_network: Option, bls_workers: Option, block_authentication: &Option, + starfish_rbc_dag_shadow: bool, ) -> eyre::Result<(NodeParameters, ClientParameters)> { let mut node_parameters = match &settings.node_parameters_path { Some(path) => NodeParameters::load(path).wrap_err("Failed to load node's parameters")?, @@ -862,6 +867,9 @@ fn load_benchmark_configs( if block_authentication.is_some() { node_parameters.block_authentication = block_authentication.clone(); } + if starfish_rbc_dag_shadow { + node_parameters.starfish_rbc_dag_shadow = true; + } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1042,6 +1050,7 @@ async fn run( .wrap_err("Failed to crate testbed")?; let block_authentication = opts.block_authentication.clone(); + let starfish_rbc_dag_shadow = opts.starfish_rbc_dag_shadow; match opts.operation { Operation::Testbed { action } => match action { // Display the current status of the testbed. @@ -1239,6 +1248,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1406,6 +1416,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1613,6 +1624,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1779,6 +1791,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1985,6 +1998,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -2330,12 +2344,14 @@ mod tests { "benchmark", "--block-authentication", "mac", + "--starfish-rbc-dag-shadow", "--protocols", "starfish-rbc", ]) .unwrap(); assert_eq!(opts.block_authentication.as_deref(), Some("mac")); + assert!(opts.starfish_rbc_dag_shadow); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); }; diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 017ee561..d4a78d4d 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -15,6 +15,9 @@ use itertools::Itertools; use prettytable::{Table, row}; use prometheus_parse::Scrape; use serde::{Deserialize, Serialize}; +use starfish_core::metrics::{ + STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, +}; use crate::{ benchmark::{BenchmarkParameters, BenchmarkRunSummary, PercentileSummary}, @@ -226,6 +229,68 @@ impl Measurement { } _ => panic!("Unexpected scraped value: '{x}'"), }, + x if matches!( + x.as_str(), + "starfish_rbc_dag_shadow_inputs_total" + | "starfish_rbc_dag_shadow_delivery_comparisons_total" + ) => + { + match sample.value { + prometheus_parse::Value::Counter(value) => { + let shadow_bucket = if x + == "starfish_rbc_dag_shadow_delivery_comparisons_total" + { + sample + .labels + .get("outcome") + .map(str::to_owned) + .unwrap_or(label) + } else { + match (sample.labels.get("kind"), sample.labels.get("outcome")) { + (Some(kind), Some(outcome)) => format!("{kind},{outcome}"), + _ => label, + } + }; + measurement + .count_buckets + .insert(shadow_bucket, value as usize); + measurement.count = measurement.count_buckets.values().sum(); + } + _ => panic!("Unexpected scraped value: '{x}'"), + } + } + x if matches!( + x.as_str(), + "starfish_rbc_dag_shadow_wal_durable_batches_total" + | "starfish_rbc_dag_shadow_wal_durable_records_total" + | "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total" + ) => + { + match sample.value { + prometheus_parse::Value::Counter(value) => { + measurement.count = value as usize; + measurement.scalar = value; + } + _ => panic!("Unexpected scraped value: '{x}'"), + } + } + x if matches!( + x.as_str(), + "starfish_rbc_dag_shadow_wal_replayed_batches" + | "starfish_rbc_dag_shadow_pending_recovery" + | "starfish_rbc_dag_shadow_comparison_valid" + | "starfish_rbc_dag_shadow_unpaired_direct" + | "starfish_rbc_dag_shadow_unpaired_shadow" + | "starfish_rbc_dag_shadow_unpaired_max_round_lag" + ) => + { + match sample.value { + prometheus_parse::Value::Gauge(value) => { + measurement.scalar = value; + } + _ => panic!("Unexpected scraped value: '{x}'"), + } + } _ => { measurements.remove(&sample.metric); } @@ -293,6 +358,10 @@ pub struct MeasurementsCollection { /// Validators that contributed at least one parsed measurement sample. #[serde(default)] observed_scrapers: BTreeSet, + /// Validators represented only by a synthetic missing-final-scrape + /// invalidation marker, not by a successfully parsed metrics response. + #[serde(default)] + synthetic_only_scrapers: BTreeSet, } impl MeasurementsCollection { @@ -308,6 +377,7 @@ impl MeasurementsCollection { db_sizes: Vec::new(), ready_nodes_at_boot, observed_scrapers: BTreeSet::new(), + synthetic_only_scrapers: BTreeSet::new(), } } @@ -321,6 +391,7 @@ impl MeasurementsCollection { /// Add a new measurement to the collection. pub fn add(&mut self, scraper_id: ScraperId, label: String, measurement: Measurement) { self.observed_scrapers.insert(scraper_id); + self.synthetic_only_scrapers.remove(&scraper_id); self.data .entry(label) .or_default() @@ -337,6 +408,33 @@ impl MeasurementsCollection { self.ready_nodes_at_boot = ready_nodes_at_boot.min(self.parameters.nodes); } + /// Record that an initially-live validator did not provide a usable final + /// shadow scrape. Appending an explicit invalid observation prevents an + /// earlier successful scrape from being mistaken for fresh final evidence. + pub fn mark_shadow_final_scrape_missing(&mut self, scraper_id: ScraperId) { + let timestamp = self + .data + .values() + .filter_map(|by_scraper| by_scraper.get(&scraper_id)) + .filter_map(|series| series.last()) + .map(Measurement::timestamp) + .max() + .unwrap_or_default(); + if !self.observed_scrapers.contains(&scraper_id) { + self.synthetic_only_scrapers.insert(scraper_id); + } + self.data + .entry("starfish_rbc_dag_shadow_comparison_valid".to_owned()) + .or_default() + .entry(scraper_id) + .or_default() + .push(Measurement { + timestamp, + scalar: 0.0, + ..Measurement::default() + }); + } + /// Get all labels. pub fn labels(&self) -> impl Iterator { self.data.keys() @@ -386,6 +484,7 @@ impl MeasurementsCollection { self.data .values() .flat_map(|samples| samples.keys().copied()) + .filter(|scraper_id| !self.synthetic_only_scrapers.contains(scraper_id)) .collect::>() .len() } @@ -399,6 +498,133 @@ impl MeasurementsCollection { .unwrap_or_default() } + /// Sum a Prometheus counter bucket across scrapers while preserving work + /// observed before a process-local counter reset. + fn sum_count_bucket_increments(&self, label: &str, bucket: &str) -> usize { + self.data + .get(label) + .into_iter() + .flat_map(|by_scraper| by_scraper.values()) + .map(|series| { + let mut previous = 0; + let mut total = 0; + for measurement in series { + let current = measurement.count_buckets.get(bucket).copied().unwrap_or(0); + total += if current >= previous { + current - previous + } else { + current + }; + previous = current; + } + total + }) + .sum() + } + + fn sum_latest_scalar_as_usize(&self, label: &str) -> usize { + self.latest_measurements(label) + .into_iter() + .map(|measurement| measurement.scalar.max(0.0) as usize) + .sum() + } + + /// Sum scalar Prometheus counter increments across scrapers and resets. + fn sum_scalar_counter_increments(&self, label: &str) -> usize { + self.data + .get(label) + .into_iter() + .flat_map(|by_scraper| by_scraper.values()) + .map(|series| { + let mut previous = 0.0; + let mut total = 0.0; + for measurement in series { + let current = measurement.scalar.max(0.0); + total += if current >= previous { + current - previous + } else { + current + }; + previous = current; + } + total as usize + }) + .sum() + } + + fn scraper_series(&self, label: &str, scraper_id: ScraperId) -> Option<&[Measurement]> { + self.data.get(label)?.get(&scraper_id).map(Vec::as_slice) + } + + fn gauge_always_equals(&self, label: &str, scraper_id: ScraperId, expected: f64) -> bool { + self.scraper_series(label, scraper_id) + .is_some_and(|series| { + !series.is_empty() + && series + .iter() + .all(|measurement| measurement.scalar == expected) + }) + } + + fn scalar_counter_is_monotonic_and_positive(&self, label: &str, scraper_id: ScraperId) -> bool { + let Some(series) = self.scraper_series(label, scraper_id) else { + return false; + }; + let monotonic = series + .windows(2) + .all(|window| window[1].scalar >= window[0].scalar); + monotonic + && series + .last() + .is_some_and(|measurement| measurement.scalar > 0.0) + } + + fn count_bucket_is_monotonic_and_positive( + &self, + label: &str, + scraper_id: ScraperId, + bucket: &str, + ) -> bool { + let Some(series) = self.scraper_series(label, scraper_id) else { + return false; + }; + let values = series + .iter() + .map(|measurement| measurement.count_buckets.get(bucket).copied().unwrap_or(0)) + .collect::>(); + values.windows(2).all(|window| window[1] >= window[0]) + && values.last().is_some_and(|value| *value > 0) + } + + fn count_bucket_is_always_zero( + &self, + label: &str, + scraper_id: ScraperId, + bucket: &str, + ) -> bool { + self.scraper_series(label, scraper_id).is_none_or(|series| { + series + .iter() + .all(|measurement| measurement.count_buckets.get(bucket).copied().unwrap_or(0) == 0) + }) + } + + fn latest_scalar_equals(&self, label: &str, scraper_id: ScraperId, expected: f64) -> bool { + self.scraper_series(label, scraper_id) + .and_then(|series| series.last()) + .is_some_and(|measurement| measurement.scalar == expected) + } + + fn gauge_always_at_most(&self, label: &str, scraper_id: ScraperId, maximum: f64) -> bool { + self.scraper_series(label, scraper_id) + .is_some_and(|series| { + !series.is_empty() + && series.iter().all(|measurement| { + measurement.scalar >= 0.0 && measurement.scalar <= maximum + }) + }) + } + /// Aggregate the benchmark duration of multiple data points by taking the /// max. pub fn benchmark_duration(&self) -> Duration { @@ -634,6 +860,121 @@ impl MeasurementsCollection { } }) .collect(); + let shadow_comparison_enabled = self.parameters.consensus_protocol == "starfish-rbc" + && self.parameters.node_parameters.starfish_rbc_dag_shadow; + let shadow_valid_scrapers = self + .data + .get("starfish_rbc_dag_shadow_comparison_valid") + .map(|by_scraper| { + by_scraper + .keys() + .copied() + .filter(|scraper_id| { + self.gauge_always_equals( + "starfish_rbc_dag_shadow_comparison_valid", + *scraper_id, + 1.0, + ) + }) + .collect::>() + }) + .unwrap_or_default(); + let shadow_comparison_valid_nodes = shadow_valid_scrapers.len(); + let shadow_delivery_matches = self.sum_count_bucket_increments( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + "match", + ); + let shadow_delivery_mismatches = ["mismatch", "direct_only", "shadow_only"] + .into_iter() + .map(|bucket| { + self.sum_count_bucket_increments( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + bucket, + ) + }) + .sum(); + let shadow_delivery_ambiguous = self.sum_count_bucket_increments( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + "ambiguous", + ); + let shadow_direct_deliveries = self + .sum_count_bucket_increments("starfish_rbc_dag_shadow_inputs_total", "delivery,direct"); + let shadow_deliveries = self + .sum_count_bucket_increments("starfish_rbc_dag_shadow_inputs_total", "delivery,shadow"); + let shadow_wal_durable_records = + self.sum_scalar_counter_increments("starfish_rbc_dag_shadow_wal_durable_records_total"); + let shadow_pending_recovery = + self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_pending_recovery"); + let shadow_unpaired_direct = + self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_unpaired_direct"); + let shadow_unpaired_shadow = + self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_unpaired_shadow"); + let shadow_unpaired_max_round_lag = self.max_result( + "starfish_rbc_dag_shadow_unpaired_max_round_lag", + |measurement| measurement.scalar.max(0.0) as usize, + ); + let maximum_unpaired_per_node = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR + .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); + let every_shadow_scraper_has_coverage = shadow_valid_scrapers.iter().all(|scraper_id| { + self.count_bucket_is_monotonic_and_positive( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + "delivery,direct", + ) && self.count_bucket_is_monotonic_and_positive( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + "delivery,shadow", + ) && self.count_bucket_is_monotonic_and_positive( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + *scraper_id, + "match", + ) && self.count_bucket_is_always_zero( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + *scraper_id, + "mismatch", + ) && self.count_bucket_is_always_zero( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + *scraper_id, + "direct_only", + ) && self.count_bucket_is_always_zero( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + *scraper_id, + "shadow_only", + ) && self.count_bucket_is_always_zero( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + *scraper_id, + "ambiguous", + ) && self.count_bucket_is_monotonic_and_positive( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + "delivery,shadow", + ) && self.scalar_counter_is_monotonic_and_positive( + "starfish_rbc_dag_shadow_wal_durable_records_total", + *scraper_id, + ) && self.latest_scalar_equals( + "starfish_rbc_dag_shadow_pending_recovery", + *scraper_id, + 0.0, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_unpaired_direct", + *scraper_id, + maximum_unpaired_per_node as f64, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_unpaired_shadow", + *scraper_id, + maximum_unpaired_per_node as f64, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_unpaired_max_round_lag", + *scraper_id, + STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG as f64, + ) + }); + let expected_shadow_nodes = self.ready_nodes_at_boot(); + let shadow_comparison_valid = shadow_comparison_enabled + && shadow_comparison_valid_nodes == expected_shadow_nodes + && every_shadow_scraper_has_coverage + && shadow_delivery_mismatches == 0 + && shadow_delivery_ambiguous == 0; BenchmarkRunSummary { protocol: self.parameters.consensus_protocol.clone(), @@ -667,6 +1008,19 @@ impl MeasurementsCollection { .average_latest_weighted_scalar("proposed_header_size_bytes"), ready_nodes_at_boot: self.ready_nodes_at_boot(), metrics_contributors: self.metrics_contributors(), + shadow_comparison_enabled, + shadow_comparison_valid, + shadow_comparison_valid_nodes, + shadow_direct_deliveries, + shadow_deliveries, + shadow_delivery_matches, + shadow_delivery_mismatches, + shadow_delivery_ambiguous, + shadow_wal_durable_records, + shadow_pending_recovery, + shadow_unpaired_direct, + shadow_unpaired_shadow, + shadow_unpaired_max_round_lag, } } @@ -716,6 +1070,29 @@ impl MeasurementsCollection { table.add_row(row![b->"Duration:", format!("{:.1} s", duration.as_secs_f64())]); table.add_row(row![b->"TPS:", format!("{:.2} tx/s", summary.tps)]); table.add_row(row![b->"BPS:", format!("{:.2} blocks/s", summary.bps)]); + if summary.shadow_comparison_enabled { + table.add_row(row![ + b->"RBC-DAG shadow:", + format!( + "valid={} ({}/{} validators), direct={}, shadow={}, matches={}, \ + mismatches={}, ambiguous={}, WAL records={}, pending recovery={}, \ + unpaired direct/shadow={}/{}, max unpaired lag={} rounds", + summary.shadow_comparison_valid, + summary.shadow_comparison_valid_nodes, + summary.ready_nodes_at_boot, + summary.shadow_direct_deliveries, + summary.shadow_deliveries, + summary.shadow_delivery_matches, + summary.shadow_delivery_mismatches, + summary.shadow_delivery_ambiguous, + summary.shadow_wal_durable_records, + summary.shadow_pending_recovery, + summary.shadow_unpaired_direct, + summary.shadow_unpaired_shadow, + summary.shadow_unpaired_max_round_lag, + ) + ]); + } table.add_row(row![ b->"End-to-end latency:", format!( @@ -792,6 +1169,106 @@ mod test { use super::{BenchmarkParameters, Measurement, MeasurementsCollection}; use crate::protocol::test_protocol_metrics::TestProtocolMetrics; + fn shadow_benchmark_parameters(nodes: usize) -> BenchmarkParameters { + let mut parameters = BenchmarkParameters::new_for_tests(); + parameters.nodes = nodes; + parameters.consensus_protocol = "starfish-rbc".to_owned(); + parameters.node_parameters.starfish_rbc_dag_shadow = true; + parameters + } + + #[allow(clippy::too_many_arguments)] + fn add_shadow_snapshot( + collection: &mut MeasurementsCollection, + scraper_id: usize, + comparison_valid: f64, + direct_deliveries: usize, + shadow_deliveries: usize, + matches: usize, + mismatches: usize, + direct_only: usize, + shadow_only: usize, + ambiguous: usize, + wal_durable_records: f64, + ) { + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_comparison_valid".to_owned(), + Measurement { + scalar: comparison_valid, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_inputs_total".to_owned(), + Measurement { + count_buckets: HashMap::from([ + ("delivery,direct".to_owned(), direct_deliveries), + ("delivery,shadow".to_owned(), shadow_deliveries), + ]), + count: direct_deliveries + shadow_deliveries, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_delivery_comparisons_total".to_owned(), + Measurement { + count_buckets: HashMap::from([ + ("match".to_owned(), matches), + ("mismatch".to_owned(), mismatches), + ("direct_only".to_owned(), direct_only), + ("shadow_only".to_owned(), shadow_only), + ("ambiguous".to_owned(), ambiguous), + ]), + count: matches + mismatches + direct_only + shadow_only + ambiguous, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), + Measurement { + count: wal_durable_records as usize, + scalar: wal_durable_records, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_pending_recovery".to_owned(), + Measurement::default(), + ); + add_shadow_backlog_snapshot(collection, scraper_id, 0, 0, 0); + } + + fn add_shadow_backlog_snapshot( + collection: &mut MeasurementsCollection, + scraper_id: usize, + unpaired_direct: usize, + unpaired_shadow: usize, + max_round_lag: usize, + ) { + for (label, value) in [ + ("starfish_rbc_dag_shadow_unpaired_direct", unpaired_direct), + ("starfish_rbc_dag_shadow_unpaired_shadow", unpaired_shadow), + ( + "starfish_rbc_dag_shadow_unpaired_max_round_lag", + max_round_lag, + ), + ] { + collection.add( + scraper_id, + label.to_owned(), + Measurement { + scalar: value as f64, + ..Measurement::default() + }, + ); + } + } + #[test] fn average_latency() { let data = Measurement { @@ -941,6 +1418,199 @@ bytes_sent_total 6284648 assert_eq!(data.timestamp.as_secs(), 300); } + #[test] + fn prometheus_parse_preserves_shadow_verdict_and_coverage() { + let report = r#" +# TYPE benchmark_duration counter +benchmark_duration 30 +# TYPE starfish_rbc_dag_shadow_comparison_valid gauge +starfish_rbc_dag_shadow_comparison_valid{node="node-0"} 1 +# TYPE starfish_rbc_dag_shadow_delivery_comparisons_total counter +starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="match"} 7 +starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="mismatch"} 0 +starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="ambiguous"} 0 +# TYPE starfish_rbc_dag_shadow_inputs_total counter +starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="shadow"} 7 +starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="direct"} 7 +# TYPE starfish_rbc_dag_shadow_wal_durable_records_total counter +starfish_rbc_dag_shadow_wal_durable_records_total{node="node-0"} 42 +# TYPE starfish_rbc_dag_shadow_wal_replayed_batches gauge +starfish_rbc_dag_shadow_wal_replayed_batches{node="node-0"} 3 +# TYPE starfish_rbc_dag_shadow_pending_recovery gauge +starfish_rbc_dag_shadow_pending_recovery{node="node-0"} 0 +# TYPE starfish_rbc_dag_shadow_unpaired_direct gauge +starfish_rbc_dag_shadow_unpaired_direct{node="node-0"} 2 +# TYPE starfish_rbc_dag_shadow_unpaired_shadow gauge +starfish_rbc_dag_shadow_unpaired_shadow{node="node-0"} 1 +# TYPE starfish_rbc_dag_shadow_unpaired_max_round_lag gauge +starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 +"#; + + let measurements = Measurement::from_prometheus::(report); + assert_eq!( + measurements["starfish_rbc_dag_shadow_comparison_valid"].scalar, + 1.0 + ); + assert_eq!( + measurements["starfish_rbc_dag_shadow_delivery_comparisons_total"].count_buckets["match"], + 7 + ); + assert_eq!( + measurements["starfish_rbc_dag_shadow_inputs_total"].count_buckets["delivery,shadow"], + 7 + ); + assert_eq!( + measurements["starfish_rbc_dag_shadow_wal_durable_records_total"].scalar, + 42.0 + ); + assert_eq!( + measurements["starfish_rbc_dag_shadow_wal_replayed_batches"].scalar, + 3.0 + ); + + let mut parameters = BenchmarkParameters::new_for_tests(); + parameters.nodes = 1; + parameters.consensus_protocol = "starfish-rbc".to_owned(); + parameters.node_parameters.starfish_rbc_dag_shadow = true; + let mut collection = MeasurementsCollection::new(parameters); + for (label, measurement) in measurements { + collection.add(0, label, measurement); + } + let summary = collection.benchmark_run_summary(); + assert!(summary.shadow_comparison_enabled); + assert!(summary.shadow_comparison_valid); + assert_eq!(summary.shadow_comparison_valid_nodes, 1); + assert_eq!(summary.shadow_direct_deliveries, 7); + assert_eq!(summary.shadow_deliveries, 7); + assert_eq!(summary.shadow_delivery_matches, 7); + assert_eq!(summary.shadow_wal_durable_records, 42); + assert_eq!(summary.shadow_unpaired_direct, 2); + assert_eq!(summary.shadow_unpaired_shadow, 1); + assert_eq!(summary.shadow_unpaired_max_round_lag, 1); + } + + #[test] + fn shadow_verdict_remains_invalid_after_historical_failure_and_counter_reset() { + let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); + + // The first scrape records an invalid gauge and every non-match + // comparison category. The second scrape deliberately looks clean, + // including reset comparison counters, so a latest-value-only verdict + // would incorrectly accept the run. + add_shadow_snapshot(&mut collection, 0, 0.0, 1, 1, 1, 1, 1, 1, 1, 1.0); + add_shadow_snapshot(&mut collection, 0, 1.0, 2, 2, 2, 0, 0, 0, 0, 2.0); + + assert!(!collection.gauge_always_equals( + "starfish_rbc_dag_shadow_comparison_valid", + 0, + 1.0, + )); + for bucket in ["mismatch", "direct_only", "shadow_only", "ambiguous"] { + assert!(!collection.count_bucket_is_always_zero( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + 0, + bucket, + )); + } + + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_delivery_mismatches, 3); + assert_eq!(summary.shadow_delivery_ambiguous, 1); + assert!(!summary.shadow_comparison_valid); + } + + #[test] + fn missing_final_shadow_scrape_invalidates_a_previously_valid_snapshot() { + let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); + add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); + assert!(collection.benchmark_run_summary().shadow_comparison_valid); + + collection.mark_shadow_final_scrape_missing(0); + + let validity = collection + .scraper_series("starfish_rbc_dag_shadow_comparison_valid", 0) + .unwrap(); + assert_eq!(validity.last().unwrap().scalar_value(), 0.0); + assert!(!collection.benchmark_run_summary().shadow_comparison_valid); + } + + #[test] + fn synthetic_missing_final_marker_is_not_a_metrics_contributor() { + let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); + + collection.mark_shadow_final_scrape_missing(0); + + assert_eq!(collection.metrics_contributors(), 0); + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_comparison_valid_nodes, 0); + assert!(!summary.shadow_comparison_valid); + } + + #[test] + fn shadow_verdict_requires_delivery_and_wal_coverage_from_every_scraper() { + let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(2)); + add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); + add_shadow_snapshot(&mut collection, 1, 1.0, 0, 0, 0, 0, 0, 0, 0, 0.0); + + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_comparison_valid_nodes, 2); + assert_eq!(summary.shadow_direct_deliveries, 4); + assert_eq!(summary.shadow_deliveries, 4); + assert_eq!(summary.shadow_delivery_matches, 4); + assert_eq!(summary.shadow_wal_durable_records, 8); + assert!(!summary.shadow_comparison_valid); + } + + #[test] + fn shadow_verdict_accepts_a_bounded_live_pipeline_tail() { + for (direct_deliveries, shadow_deliveries, matches) in [(5, 4, 4), (5, 5, 4)] { + let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); + add_shadow_snapshot( + &mut collection, + 0, + 1.0, + direct_deliveries, + shadow_deliveries, + matches, + 0, + 0, + 0, + 0, + 8.0, + ); + add_shadow_backlog_snapshot( + &mut collection, + 0, + direct_deliveries - matches, + shadow_deliveries - matches, + 1, + ); + + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_direct_deliveries, direct_deliveries); + assert_eq!(summary.shadow_deliveries, shadow_deliveries); + assert_eq!(summary.shadow_delivery_matches, matches); + assert!(summary.shadow_comparison_valid); + } + } + + #[test] + fn shadow_verdict_rejects_excessive_or_old_unpaired_work() { + for (unpaired_direct, unpaired_shadow, max_round_lag) in [(5, 0, 1), (0, 5, 1), (1, 0, 5)] { + let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); + add_shadow_snapshot(&mut collection, 0, 1.0, 8, 7, 7, 0, 0, 0, 0, 8.0); + add_shadow_backlog_snapshot( + &mut collection, + 0, + unpaired_direct, + unpaired_shadow, + max_round_lag, + ); + + assert!(!collection.benchmark_run_summary().shadow_comparison_valid); + } + } + #[test] fn benchmark_run_summary_includes_cpu_and_percentiles() { let report = r#" diff --git a/crates/orchestrator/src/orchestrator.rs b/crates/orchestrator/src/orchestrator.rs index c5a49bf2..6ce99bd4 100644 --- a/crates/orchestrator/src/orchestrator.rs +++ b/crates/orchestrator/src/orchestrator.rs @@ -398,6 +398,23 @@ impl Orchestrator

{ (node_count as f64 * Self::MAX_TOLERATED_BOOT_FAILURE_RATIO).floor() as usize } + fn max_tolerated_boot_failures_for( + parameters: &BenchmarkParameters, + node_count: usize, + ) -> usize { + if parameters.consensus_protocol == "starfish-rbc" + && parameters.node_parameters.starfish_rbc_dag_shadow + { + // A partial shadow committee cannot produce a complete paired + // comparison. Pre-declared startup faults are already excluded + // through `skipped_node_ids`; every remaining participant must + // finish background replay before measurement begins. + 0 + } else { + Self::max_tolerated_boot_failures(node_count) + } + } + fn apt_get_noninteractive(args: &str) -> String { format!("sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get {args}") } @@ -573,7 +590,7 @@ impl Orchestrator

{ } let timeout = Self::node_boot_timeout(total); - let max_failures = Self::max_tolerated_boot_failures(total); + let max_failures = Self::max_tolerated_boot_failures_for(parameters, total); let start = Instant::now(); display::status(format!("ready 0/{total}")); @@ -1076,6 +1093,13 @@ impl Orchestrator

{ let final_metrics_commands = self .protocol_commands .nodes_final_metrics_command(nodes.clone(), parameters); + let initial_live_scraper_ids: HashSet<_> = nodes + .iter() + .filter(|node| !killed_node_ids.contains(&node.id)) + .filter_map(|node| node_indices.get(&node.id).copied()) + .collect(); + let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" + && parameters.node_parameters.starfish_rbc_dag_shadow; let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1165,6 +1189,7 @@ impl Orchestrator

{ let mut final_instances = final_metrics_commands; final_instances.retain(|(instance, _)| !killed_node_ids.contains(&instance.id)); + let mut fresh_final_shadow_scrapers = HashSet::new(); if !final_instances.is_empty() { let expected_nodes = final_instances.len(); let stdio = self @@ -1175,35 +1200,56 @@ impl Orchestrator

{ ) .await; - if stdio.is_empty() { + if stdio.is_empty() && !shadow_final_scrape_required { display::warn( "Final metrics scrape failed for all reachable nodes; \ reporting the last successful samples", ); } else { - let successful_ids: HashSet<_> = stdio - .iter() - .map(|(instance, _)| instance.id.clone()) - .collect(); - let missed = expected_nodes.saturating_sub(successful_ids.len()); - if missed != 0 { - display::warn(format!( - "Final metrics scrape missed {missed} of {expected_nodes} nodes; \ - reporting partial results", - )); + if !shadow_final_scrape_required { + let successful_ids: HashSet<_> = stdio + .iter() + .map(|(instance, _)| instance.id.clone()) + .collect(); + let missed = expected_nodes.saturating_sub(successful_ids.len()); + if missed != 0 { + display::warn(format!( + "Final metrics scrape missed {missed} of {expected_nodes} nodes; \ + reporting partial results", + )); + } } - for (instance, (stdout, _stderr)) in &stdio { let Some(i) = node_indices.get(&instance.id).copied() else { continue; }; - for (label, measurement) in Measurement::from_prometheus::

(stdout) { + let parsed = Measurement::from_prometheus::

(stdout); + if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { + fresh_final_shadow_scrapers.insert(i); + } + for (label, measurement) in parsed { aggregator.add(i, label, measurement); } } - aggregator.save(&self.suite_results_dir); } } + if shadow_final_scrape_required { + let missed = initial_live_scraper_ids + .difference(&fresh_final_shadow_scrapers) + .count(); + if missed != 0 { + display::warn(format!( + "Final metrics scrape was missing or lacked shadow validity evidence for \ + {missed} of {} initially-live nodes; missing final evidence invalidates stale \ + shadow results", + initial_live_scraper_ids.len(), + )); + } + for scraper_id in initial_live_scraper_ids.difference(&fresh_final_shadow_scrapers) { + aggregator.mark_shadow_final_scrape_missing(*scraper_id); + } + } + aggregator.save(&self.suite_results_dir); display::done(); Ok(aggregator) @@ -1379,6 +1425,13 @@ impl Orchestrator

{ let final_metrics_commands = self .protocol_commands .nodes_final_metrics_command(nodes.clone(), parameters); + let initial_live_scraper_ids: HashSet<_> = nodes + .iter() + .filter(|node| !killed_node_ids.contains(&node.id)) + .filter_map(|node| node_indices.get(&node.id).copied()) + .collect(); + let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" + && parameters.node_parameters.starfish_rbc_dag_shadow; let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1759,7 +1812,9 @@ impl Orchestrator

{ let mut final_instances = final_metrics_commands; final_instances.retain(|(instance, _)| !killed_node_ids.contains(&instance.id)); + let mut fresh_final_shadow_scrapers = HashSet::new(); if !final_instances.is_empty() { + let expected_nodes = final_instances.len(); let stdio = self .execute_per_instance_best_effort( final_instances, @@ -1767,16 +1822,54 @@ impl Orchestrator

{ "Final stability metrics scrape", ) .await; + if stdio.is_empty() && !shadow_final_scrape_required { + display::warn( + "Final stability metrics scrape failed for all reachable nodes; \ + reporting the last successful samples", + ); + } else if !shadow_final_scrape_required { + let successful_ids: HashSet<_> = stdio + .iter() + .map(|(instance, _)| instance.id.clone()) + .collect(); + let missed = expected_nodes.saturating_sub(successful_ids.len()); + if missed != 0 { + display::warn(format!( + "Final stability metrics scrape missed {missed} of {expected_nodes} nodes; \ + reporting partial results", + )); + } + } for (instance, (stdout, _stderr)) in &stdio { let Some(i) = node_indices.get(&instance.id).copied() else { continue; }; - for (label, measurement) in Measurement::from_prometheus::

(stdout) { + let parsed = Measurement::from_prometheus::

(stdout); + if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { + fresh_final_shadow_scrapers.insert(i); + } + for (label, measurement) in parsed { aggregator.add(i, label, measurement); } } - aggregator.save(&self.suite_results_dir); } + if shadow_final_scrape_required { + let missed = initial_live_scraper_ids + .difference(&fresh_final_shadow_scrapers) + .count(); + if missed != 0 { + display::warn(format!( + "Final stability scrape was missing or lacked shadow validity evidence for \ + {missed} of {} initially-live nodes; missing final evidence invalidates stale \ + shadow results", + initial_live_scraper_ids.len(), + )); + } + for scraper_id in initial_live_scraper_ids.difference(&fresh_final_shadow_scrapers) { + aggregator.mark_shadow_final_scrape_missing(*scraper_id); + } + } + aggregator.save(&self.suite_results_dir); display::done(); Ok((aggregator, report)) @@ -2201,6 +2294,7 @@ impl Orchestrator

{ #[cfg(test)] mod tests { use super::Orchestrator; + use crate::benchmark::BenchmarkParameters; use crate::protocol::starfish::StarfishProtocol; use crate::{client::Instance, faults::FaultsType}; @@ -2232,6 +2326,18 @@ mod tests { ); } + #[test] + fn shadow_readiness_requires_every_non_skipped_validator() { + let mut parameters = BenchmarkParameters::new_for_tests(); + parameters.consensus_protocol = "starfish-rbc".to_owned(); + parameters.node_parameters.starfish_rbc_dag_shadow = true; + + assert_eq!( + Orchestrator::::max_tolerated_boot_failures_for(¶meters, 10), + 0 + ); + } + #[test] fn startup_permanent_faults_are_left_down_from_boot() { let nodes = (0..100) diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 11a7fe72..e5bcdf1f 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -15,7 +15,10 @@ use starfish_core::{ types::AuthorityIndex, }; -use super::{BINARY_PATH, ProtocolCommands, ProtocolMetrics, ProtocolParameters}; +use super::{ + BINARY_PATH, METRICS_CURL_CONNECT_TIMEOUT_SECS, METRICS_CURL_MAX_TIME_SECS, ProtocolCommands, + ProtocolMetrics, ProtocolParameters, +}; use crate::{benchmark::BenchmarkParameters, client::Instance, settings::Settings}; #[derive(Clone, Serialize, Deserialize, Default)] @@ -244,6 +247,50 @@ impl ProtocolMetrics for StarfishProtocol { instances.into_iter().zip(metrics_paths).collect() } + + fn nodes_readiness_command( + &self, + instances: I, + parameters: &BenchmarkParameters, + ) -> Vec<(Instance, String)> + where + I: IntoIterator, + { + if parameters.consensus_protocol != "starfish-rbc" + || !parameters.node_parameters.starfish_rbc_dag_shadow + { + return self + .nodes_metrics_path(instances, parameters) + .into_iter() + .map(|(instance, path)| { + ( + instance, + format!( + "curl -sf -o /dev/null --compressed --connect-timeout \ + {METRICS_CURL_CONNECT_TIMEOUT_SECS} --max-time \ + {METRICS_CURL_MAX_TIME_SECS} {path}" + ), + ) + }) + .collect(); + } + + self.nodes_metrics_path(instances, parameters) + .into_iter() + .map(|(instance, path)| { + ( + instance, + format!( + "curl --silent --show-error --fail --compressed --connect-timeout \ + {METRICS_CURL_CONNECT_TIMEOUT_SECS} --max-time \ + {METRICS_CURL_MAX_TIME_SECS} {path} | grep -Eq \ + '^starfish_rbc_dag_shadow_comparison_valid(\\{{[^}}]*\\}})? \ + 1(\\.0)?$'" + ), + ) + }) + .collect() + } } impl StarfishProtocol { @@ -265,6 +312,12 @@ impl StarfishProtocol { ) -> StarfishNodeParameters { if consensus_protocol == "starfish-rbc" { node_parameters.refresh_starfish_rbc_protocol_instance(); + } else { + // The CLI flag is global to a multi-protocol benchmark plan, but + // the shadow is meaningful only for the Starfish-RBC member. Do + // not let it make Sailfish++ or another comparison member fail + // validator configuration. + node_parameters.starfish_rbc_dag_shadow = false; } node_parameters } @@ -299,28 +352,57 @@ impl StarfishProtocol { #[cfg(test)] mod tests { - use super::{StarfishNodeParameters, StarfishProtocol}; + use super::{ProtocolMetrics, StarfishNodeParameters, StarfishProtocol}; + use crate::{benchmark::BenchmarkParameters, client::Instance}; + use starfish_core::config::NodeParameters; #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { - let parameters = StarfishProtocol::node_parameters_for_genesis( - "starfish-rbc", - StarfishNodeParameters::default(), - ); + let shared_parameters = StarfishNodeParameters(NodeParameters { + starfish_rbc_dag_shadow: true, + ..NodeParameters::default() + }); + let parameters = + StarfishProtocol::node_parameters_for_genesis("starfish-rbc", shared_parameters); assert!( parameters .starfish_rbc_protocol_instance .is_some_and(|instance| instance != [0; 32]) ); + assert!(parameters.starfish_rbc_dag_shadow); } #[test] fn non_rbc_genesis_does_not_need_a_protocol_instance() { - let parameters = StarfishProtocol::node_parameters_for_genesis( - "starfish", - StarfishNodeParameters::default(), - ); + let shared_parameters = StarfishNodeParameters(NodeParameters { + starfish_rbc_dag_shadow: true, + ..NodeParameters::default() + }); + let parameters = + StarfishProtocol::node_parameters_for_genesis("sailfish++", shared_parameters); assert_eq!(parameters.starfish_rbc_protocol_instance, None); + assert!( + !parameters.starfish_rbc_dag_shadow, + "a global shadow flag must not leak into non-RBC comparison members" + ); + } + + #[test] + fn shadow_readiness_waits_for_completed_background_replay() { + let protocol = StarfishProtocol { + working_dir: std::path::PathBuf::from("benchmark"), + }; + let mut parameters = BenchmarkParameters::new_for_tests(); + parameters.consensus_protocol = "starfish-rbc".to_owned(); + parameters.node_parameters.starfish_rbc_dag_shadow = true; + let command = protocol + .nodes_readiness_command(vec![Instance::new_for_test("1".into())], ¶meters) + .pop() + .unwrap() + .1; + + assert!(command.contains("starfish_rbc_dag_shadow_comparison_valid")); + assert!(command.contains("grep -Eq")); } #[test] diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 34bf365c..63b341e1 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -67,6 +67,11 @@ pub struct NodeParameters { /// other protocols. #[serde(default)] pub starfish_rbc_protocol_instance: Option<[u8; 32]>, + /// Run the persisted Starfish-RBC-DAG carrier implementation alongside + /// the authoritative direct Starfish-RBC service. Shadow delivery is + /// observational only and cannot affect the DAG, pacemaker, or commits. + #[serde(default)] + pub starfish_rbc_dag_shadow: bool, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -142,6 +147,7 @@ impl Default for NodeParameters { dissemination_mode: DisseminationMode::default(), block_authentication: None, starfish_rbc_protocol_instance: None, + starfish_rbc_dag_shadow: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), @@ -375,6 +381,10 @@ impl NodePrivateConfig { pub fn rocksdb(&self) -> PathBuf { self.storage_path.join("rocksdb") } + + pub fn starfish_rbc_dag_shadow_wal(&self) -> PathBuf { + self.storage_path.join("starfish-rbc-dag-shadow-v1.wal") + } } impl ImportExport for NodePrivateConfig {} @@ -387,6 +397,7 @@ mod tests { fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { let mut parameters: NodeParameters = serde_yaml::from_str("{}").unwrap(); assert_eq!(parameters.starfish_rbc_protocol_instance, None); + assert!(!parameters.starfish_rbc_dag_shadow); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); assert_ne!(protocol_instance, [0; 32]); @@ -397,6 +408,7 @@ mod tests { decoded.starfish_rbc_protocol_instance, Some(protocol_instance) ); + assert!(!decoded.starfish_rbc_dag_shadow); } } diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 7324c25f..0b9a53f0 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -504,7 +504,16 @@ mod tests { recovered, None, ); - let syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None, None); + let syncer = Syncer::new( + core, + false, + NoopCommitObserver, + metrics, + None, + None, + None, + None, + ); CoreThreadDispatcher::start(syncer) } diff --git a/crates/starfish-core/src/lib.rs b/crates/starfish-core/src/lib.rs index 70c7f628..15dee4b7 100644 --- a/crates/starfish-core/src/lib.rs +++ b/crates/starfish-core/src/lib.rs @@ -31,6 +31,8 @@ mod runtime; pub mod shard_reconstructor; pub mod starfish_rbc; pub mod starfish_rbc_dag; +mod starfish_rbc_dag_shadow; +mod starfish_rbc_dag_shadow_service; mod starfish_rbc_service; mod stat; mod state; diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 15004578..83487543 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -36,6 +36,11 @@ pub const BENCHMARK_DURATION: &str = "benchmark_duration"; pub const TRANSACTION_CERTIFIED_LATENCY: &str = "transaction_certified_latency"; pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; +/// Benchmark-only live-tail guards for the observational RBC-DAG shadow. +/// They are not asynchronous protocol or garbage-collection bounds. +pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR: i64 = 4; +pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; + #[derive(Clone)] pub struct Metrics { pub benchmark_duration: IntCounter, @@ -135,6 +140,21 @@ pub struct Metrics { pub network_message_bytes_sent_total: IntCounterVec, pub network_message_bytes_received_total: IntCounterVec, + // Starfish-RBC-DAG shadow instrumentation. These metrics are strictly + // observational: the shadow path never feeds the authoritative DAG or + // consensus state. + pub starfish_rbc_dag_shadow_inputs_total: IntCounterVec, + pub starfish_rbc_dag_shadow_delivery_comparisons_total: IntCounterVec, + pub starfish_rbc_dag_shadow_wal_durable_batches_total: IntCounter, + pub starfish_rbc_dag_shadow_wal_durable_records_total: IntCounter, + pub starfish_rbc_dag_shadow_wal_replayed_batches: IntGauge, + pub starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total: IntCounter, + pub starfish_rbc_dag_shadow_pending_recovery: IntGauge, + pub starfish_rbc_dag_shadow_unpaired_direct: IntGauge, + pub starfish_rbc_dag_shadow_unpaired_shadow: IntGauge, + pub starfish_rbc_dag_shadow_unpaired_max_round_lag: IntGauge, + pub starfish_rbc_dag_shadow_comparison_valid: IntGauge, + // subscription tracking pub subscribed_to_peers: IntGauge, pub subscribed_by_peers: IntGauge, @@ -516,6 +536,76 @@ impl Metrics { registry, ) .unwrap(), + starfish_rbc_dag_shadow_inputs_total: register_int_counter_vec_with_registry!( + "starfish_rbc_dag_shadow_inputs_total", + "Starfish-RBC-DAG shadow inputs, by bounded input kind and processing outcome", + &["kind", "outcome"], + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_delivery_comparisons_total: + register_int_counter_vec_with_registry!( + "starfish_rbc_dag_shadow_delivery_comparisons_total", + "Non-authoritative current-process paired direct-vs-shadow delivery observations, by outcome; unmatched observations are not mismatches", + &["outcome"], + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_wal_durable_batches_total: register_int_counter_with_registry!( + "starfish_rbc_dag_shadow_wal_durable_batches_total", + "Starfish-RBC-DAG shadow WAL batches durably synchronized", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_wal_durable_records_total: register_int_counter_with_registry!( + "starfish_rbc_dag_shadow_wal_durable_records_total", + "Starfish-RBC-DAG shadow WAL records durably synchronized", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_wal_replayed_batches: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_wal_replayed_batches", + "Starfish-RBC-DAG shadow WAL batches replayed during this process startup", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total: + register_int_counter_with_registry!( + "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total", + "Physically incomplete final Starfish-RBC-DAG shadow WAL bytes discarded during recovery", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_pending_recovery: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_pending_recovery", + "Current Starfish-RBC-DAG shadow carrier-content recoveries pending", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_unpaired_direct: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_unpaired_direct", + "Current direct-delivery slots without an observed shadow-delivery slot, including recovered shadow state", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_unpaired_shadow: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_unpaired_shadow", + "Current shadow-delivery slots first observed in this process without a direct-delivery slot observed in this process", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_unpaired_max_round_lag: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_unpaired_max_round_lag", + "Maximum round lag from a current unpaired direct or current-epoch shadow delivery slot to the newest current-process observation", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_comparison_valid: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_comparison_valid", + "State of the non-authoritative shadow observation stream (1 valid, 0 disabled/invalid, -1 starting)", + registry, + ) + .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -972,6 +1062,8 @@ impl Metrics { metrics: Vec>, reporters: Vec>, duration_secs: u64, + committee_size: usize, + starfish_rbc_dag_shadow_expected: bool, ) { let num_validators = metrics.len() as u64; @@ -1109,6 +1201,9 @@ impl Metrics { "rbc_ready", "rbc_header_request", "rbc_header_response", + "rbc_dag_shadow_carrier", + "rbc_dag_shadow_carrier_request", + "rbc_dag_shadow_carrier_response", ]; let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() @@ -1164,6 +1259,154 @@ impl Metrics { }; table.add_row(row![b->"Bandwidth efficiency:", format!("{:.2}", bandwidth_efficiency)]); + if starfish_rbc_dag_shadow_expected { + let valid_nodes = metrics + .iter() + .filter(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) + .count(); + let matches = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["match"]) + .get() + }) + .sum::(); + let mismatches = metrics + .iter() + .map(|metrics| { + ["mismatch", "direct_only", "shadow_only"] + .into_iter() + .map(|outcome| { + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&[outcome]) + .get() + }) + .sum::() + }) + .sum::(); + let ambiguous = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["ambiguous"]) + .get() + }) + .sum::(); + let shadow_deliveries = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get() + }) + .sum::(); + let direct_deliveries = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "direct"]) + .get() + }) + .sum::(); + let wal_records = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .get() + }) + .sum::(); + let pending_recovery = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_pending_recovery.get()) + .sum::(); + let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR + .saturating_mul(i64::try_from(committee_size).unwrap_or(i64::MAX)); + let every_node_has_exact_coverage = metrics.iter().all(|metrics| { + let direct = metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "direct"]) + .get(); + let shadow = metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get(); + let matched = metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["match"]) + .get(); + direct > 0 + && shadow > 0 + && matched > 0 + && metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .get() + > 0 + && metrics.starfish_rbc_dag_shadow_unpaired_direct.get() <= maximum_unpaired + && metrics.starfish_rbc_dag_shadow_unpaired_shadow.get() <= maximum_unpaired + && metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get() + <= STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG + }); + let comparison_valid = valid_nodes == metrics.len() + && every_node_has_exact_coverage + && mismatches == 0 + && ambiguous == 0 + && wal_records > 0 + && pending_recovery == 0; + + table.add_row(row![bH2->""]); + table.add_row(row![bH2->"RBC-DAG Shadow Verification"]); + table.add_row(row![ + b->"Comparison verdict:", + if comparison_valid { + "VALID".to_owned() + } else { + "INVALID — DISCARD THIS SHADOW COMPARISON".to_owned() + } + ]); + table.add_row(row![ + b->"Valid validators:", + format!("{valid_nodes}/{}", metrics.len()) + ]); + table.add_row(row![ + b->"Paired deliveries:", + format!( + "direct={direct_deliveries}, shadow={shadow_deliveries}, matches={matches}, \ + mismatches={mismatches}, ambiguous={ambiguous}" + ) + ]); + table.add_row(row![ + b->"Durability/recovery:", + format!("WAL records={wal_records}, pending recovery={pending_recovery}") + ]); + let unpaired_direct = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_unpaired_direct.get()) + .sum::(); + let unpaired_shadow = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_unpaired_shadow.get()) + .sum::(); + let max_unpaired_lag = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get()) + .max() + .unwrap_or_default(); + table.add_row(row![ + b->"Live comparison tail:", + format!( + "unpaired direct/shadow={unpaired_direct}/{unpaired_shadow}, \ + max lag={max_unpaired_lag} rounds" + ) + ]); + } + // Shard reconstruction metrics table.add_row(row![bH2->""]); table.add_row(row![bH2->"Shard Reconstruction"]); @@ -1462,3 +1705,60 @@ struct NetworkAddressTable { peer: String, address: String, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registers_starfish_rbc_dag_shadow_metrics() { + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new(®istry, None, None, None); + + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["authenticated_ingress", "accepted"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["match"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .inc_by(3); + metrics.starfish_rbc_dag_shadow_wal_replayed_batches.set(4); + metrics + .starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total + .inc_by(5); + metrics.starfish_rbc_dag_shadow_pending_recovery.set(2); + metrics.starfish_rbc_dag_shadow_unpaired_direct.set(6); + metrics.starfish_rbc_dag_shadow_unpaired_shadow.set(7); + metrics + .starfish_rbc_dag_shadow_unpaired_max_round_lag + .set(8); + metrics.starfish_rbc_dag_shadow_comparison_valid.set(1); + + let gathered = registry.gather(); + for name in [ + "starfish_rbc_dag_shadow_inputs_total", + "starfish_rbc_dag_shadow_delivery_comparisons_total", + "starfish_rbc_dag_shadow_wal_durable_batches_total", + "starfish_rbc_dag_shadow_wal_durable_records_total", + "starfish_rbc_dag_shadow_wal_replayed_batches", + "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total", + "starfish_rbc_dag_shadow_pending_recovery", + "starfish_rbc_dag_shadow_unpaired_direct", + "starfish_rbc_dag_shadow_unpaired_shadow", + "starfish_rbc_dag_shadow_unpaired_max_round_lag", + "starfish_rbc_dag_shadow_comparison_valid", + ] { + assert!( + gathered.iter().any(|family| family.get_name() == name), + "metric family {name} was not registered", + ); + } + } +} diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 75adafec..819a586d 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -4,6 +4,7 @@ use std::{ collections::{HashMap, VecDeque}, + path::PathBuf, sync::{ Arc, atomic::{AtomicU32, Ordering}, @@ -45,6 +46,14 @@ use crate::{ }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, + starfish_rbc_dag::{RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId}, + starfish_rbc_dag_shadow::{ + ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, + }, + starfish_rbc_dag_shadow_service::{ + ShadowServiceErrorV1, ShadowServiceEventV1, StarfishRbcDagShadowServiceHandleV1, + start_starfish_rbc_dag_shadow_service_v1, + }, starfish_rbc_service::{ RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, }, @@ -60,6 +69,82 @@ const MAX_FILTER_SIZE: usize = 100_000; const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); +const STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); + +/// Recover the exact locally selected Starfish-RBC chain so the persisted +/// non-authoritative shadow can reconcile a WAL that ended before the direct +/// DAG. The newest local block determines the branch when a Byzantine test +/// produced more than one value in a round. +fn recovered_local_rbc_headers( + core: &Core, +) -> Result, String> { + if !core.dag_state().consensus_protocol.is_starfish_rbc() || core.last_proposed() == 0 { + return Ok(Vec::new()); + } + + let own_authority = core.authority(); + let store = core.store(); + let mut current = core.last_own_block().clone(); + let mut reversed = Vec::with_capacity(current.round() as usize); + loop { + if current.round() == 0 { + break; + } + if current.authority() != own_authority { + return Err(format!( + "recovered local chain contains authority {} at round {} (expected {})", + current.authority(), + current.round(), + own_authority + )); + } + reversed.push( + RbcCanonicalHeader::from_block_header(current.header()).map_err(|error| { + format!( + "recovered local Starfish-RBC header {} is not canonical: {error}", + current.reference() + ) + })?, + ); + if current.round() == 1 { + break; + } + + let expected_round = current.round() - 1; + let predecessor = current + .block_references() + .iter() + .find(|reference| { + reference.authority == own_authority && reference.round == expected_round + }) + .copied() + .ok_or_else(|| { + format!( + "recovered local Starfish-RBC block {} has no own predecessor at round {}", + current.reference(), + expected_round + ) + })?; + current = core + .dag_state() + .get_blocks_at_authority_round(own_authority, expected_round) + .into_iter() + .find(|block| block.reference() == &predecessor) + .or_else(|| store.get_block(&predecessor).ok().flatten()) + .ok_or_else(|| { + format!("recovered local Starfish-RBC predecessor {predecessor} is unavailable") + })?; + } + reversed.reverse(); + Ok(reversed) +} + +fn shadow_transport_error_invalidates_comparison(error: &ShadowServiceErrorV1) -> bool { + matches!( + error, + ShadowServiceErrorV1::Overloaded { .. } | ShadowServiceErrorV1::Stopped + ) +} /// Enforce the MAC experiment's transport contract before cryptographic /// verification: @@ -741,6 +826,7 @@ struct ConnectionHandler bls_service: Option, sailfish_service: Option, starfish_rbc_service: Option, + starfish_rbc_dag_shadow_service: Option, } impl ConnectionHandler { @@ -783,6 +869,7 @@ impl ConnectionHandler ConnectionHandler ConnectionHandler { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.carrier(self.peer_id, envelope) { + if shadow_transport_error_invalidates_comparison(&error) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + } + tracing::warn!("Failed to forward RBC-DAG shadow carrier: {error}"); + } + } + } + NetworkMessage::RbcDagShadowCarrierRequest(reference) => { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.carrier_request(self.peer_id, reference) { + if shadow_transport_error_invalidates_comparison(&error) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + } + tracing::warn!("Failed to forward RBC-DAG shadow request: {error}"); + } + } + } + NetworkMessage::RbcDagShadowCarrierResponse(response) => { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.carrier_response(self.peer_id, response) { + if shadow_transport_error_invalidates_comparison(&error) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + } + tracing::warn!("Failed to forward RBC-DAG shadow response: {error}"); + } + } + } } true } @@ -1565,6 +1683,8 @@ pub struct NetworkSyncer { sf_event_task: Option>, rbc_event_task: Option>, rbc_service_task: Option>, + rbc_dag_shadow_event_task: Option>, + rbc_dag_shadow_service_task: Option>, cordial_knowledge_task: JoinHandle<()>, } @@ -1603,6 +1723,9 @@ pub struct NetworkSyncerInner { /// Central Starfish-RBC service. Connection workers only forward their /// trusted peer identity and wire payload into this single owner. pub(crate) starfish_rbc_service: Option, + /// Non-authoritative persisted embedded-RBC shadow. It emits only network + /// and metric events and can never call the core dispatcher. + pub(crate) starfish_rbc_dag_shadow_service: Option, /// Wall-clock at NetworkSyncer start; consumed by time-dependent /// Byzantine strategies (e.g. RampUpWithholding) to ramp behavior /// over a fixed schedule. @@ -1616,6 +1739,7 @@ impl NetworkSyncer mut commit_observer: C, metrics: Arc, node_parameters: NodeParameters, + starfish_rbc_dag_shadow_wal: PathBuf, partial_sig_outbox_rx: Option>, bls_cert_aggregator: Option, bls_signer: Option, @@ -1628,6 +1752,22 @@ impl NetworkSyncer let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); + let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { + match recovered_local_rbc_headers(&core) { + Ok(headers) => Some(headers), + Err(error) => { + // A partial local history would make delivery comparisons + // meaningless. Disable the whole observational run while + // allowing authoritative direct RBC to continue. + tracing::error!( + "Disabling non-authoritative RBC-DAG shadow because recovered direct headers cannot be reconciled: {error}" + ); + None + } + } + } else { + None + }; let dissemination_mode = dag_state .consensus_protocol .resolve_dissemination_mode(node_parameters.dissemination_mode); @@ -1692,6 +1832,57 @@ impl NetworkSyncer } else { (None, None, None) }; + let (starfish_rbc_dag_shadow_service, rbc_dag_shadow_event_rx, rbc_dag_shadow_service_task) = + if let Some(recovered_local_headers) = recovered_shadow_local_headers { + let protocol_instance_bytes = node_parameters + .starfish_rbc_protocol_instance + .expect("validated shadow configuration must share the direct RBC instance"); + let protocol_instance = RbcDagProtocolInstanceId::new(protocol_instance_bytes) + .expect("validated direct RBC instance must be nonzero"); + let committee_context = RbcDagCommitteeContextV1::new(committee.clone()) + .expect("validated committee must initialize the RBC-DAG shadow"); + let context = RbcDagContextV1::new_with_committee( + protocol_instance, + &committee_context, + dag_state.block_authentication_scheme, + ); + let authorizer = match dag_state.block_authentication_scheme { + BlockAuthenticationScheme::Ed25519 => { + ShadowAuthorizerV1::Ed25519(core.get_signer().clone()) + } + BlockAuthenticationScheme::MlDsa44 => { + ShadowAuthorizerV1::MlDsa44(core.get_ml_dsa_44_signer().clone()) + } + BlockAuthenticationScheme::MlDsa65 => { + ShadowAuthorizerV1::MlDsa65(core.get_ml_dsa_65_signer().clone()) + } + BlockAuthenticationScheme::MacVector => { + ShadowAuthorizerV1::MacVector(mac_keys.as_ref().clone()) + } + }; + // -1 means the background WAL replay has not completed yet; + // Ready moves this to 1 unless work was already shed (0). + metrics.starfish_rbc_dag_shadow_comparison_valid.set(-1); + match start_starfish_rbc_dag_shadow_service_v1( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + recovered_local_headers, + ) { + Ok((service, events, task)) => (Some(service), Some(events), Some(task)), + Err(error) => { + metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + tracing::error!( + "Disabling non-authoritative Starfish-RBC-DAG shadow: {error}" + ); + (None, None, None) + } + } + } else { + (None, None, None) + }; let syncer = Syncer::new( core, NetworkSyncSignals { @@ -1703,6 +1894,7 @@ impl NetworkSyncer bls_msg_tx.clone(), sf_msg_tx.clone(), starfish_rbc_service.clone(), + starfish_rbc_dag_shadow_service.clone(), ); let initial_round = syncer.core().next_block_round(); let syncer = CoreThreadDispatcher::start(syncer); @@ -1772,6 +1964,7 @@ impl NetworkSyncer soft_block_timeout: node_parameters.soft_block_timeout, sailfish_handle: sf_handle_for_inner, starfish_rbc_service: starfish_rbc_service.clone(), + starfish_rbc_dag_shadow_service: starfish_rbc_dag_shadow_service.clone(), start_time: std::time::Instant::now(), }); @@ -1781,6 +1974,7 @@ impl NetworkSyncer // clean. let rbc_event_task = rbc_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); + let rbc_metrics = metrics.clone(); handle.spawn(async move { let mut payload_encoder = ReedSolomonEncoder::new(2, 4, 2) .expect("Starfish-RBC payload encoder should be created"); @@ -1872,6 +2066,26 @@ impl NetworkSyncer .await; } RbcServiceEvent::Delivered(header) => { + if let Some(ref shadow) = + event_inner.starfish_rbc_dag_shadow_service + { + let canonical = header.header(); + let identity = ShadowDeliveryIdentityV1::new( + canonical.reference().authority, + canonical.reference().round, + canonical.transactions_commitment(), + ); + if let Err(error) = shadow.direct_delivered(identity) { + if shadow_transport_error_invalidates_comparison(&error) { + rbc_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + tracing::warn!( + "Failed to notify RBC-DAG shadow of direct delivery: {error}" + ); + } + } event_inner .syncer .apply_starfish_rbc_deliveries(vec![header]) @@ -1889,6 +2103,155 @@ impl NetworkSyncer }) }); + let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { + let event_inner = inner.clone(); + let shadow_metrics = metrics.clone(); + handle.spawn(async move { + while let Some(event) = event_rx.recv().await { + match event { + ShadowServiceEventV1::Network { recipient, message } => { + let sender = event_inner.peer_senders.read().get(&recipient).cloned(); + if let Some(sender) = sender { + match sender.try_send(message) { + Ok(()) => shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "sent"]) + .inc(), + Err(mpsc::error::TrySendError::Full(_)) => { + // The shadow is observational. It must + // shed work instead of backpressuring + // the authoritative network path. + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "dropped_backpressure"]) + .inc(); + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "disconnected"]) + .inc(); + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + } + } else { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "disconnected"]) + .inc(); + // No observation was lost: the actor retains + // local carriers and replays them when this + // peer connects. + } + } + ShadowServiceEventV1::Delivered(identity) => { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .inc(); + tracing::debug!(?identity, "RBC-DAG shadow delivered carrier"); + } + ShadowServiceEventV1::ComparisonBacklog { + unpaired_direct, + unpaired_shadow, + max_round_lag, + } => { + shadow_metrics + .starfish_rbc_dag_shadow_unpaired_direct + .set(i64::try_from(unpaired_direct).unwrap_or(i64::MAX)); + shadow_metrics + .starfish_rbc_dag_shadow_unpaired_shadow + .set(i64::try_from(unpaired_shadow).unwrap_or(i64::MAX)); + shadow_metrics + .starfish_rbc_dag_shadow_unpaired_max_round_lag + .set(i64::from(max_round_lag)); + } + ShadowServiceEventV1::Comparison(comparison) => { + let outcome = match &comparison { + ShadowDeliveryComparisonV1::Match => "match", + ShadowDeliveryComparisonV1::Mismatch { + direct_only, + shadow_only, + } if !direct_only.is_empty() && shadow_only.is_empty() => { + "direct_only" + } + ShadowDeliveryComparisonV1::Mismatch { + direct_only, + shadow_only, + } if direct_only.is_empty() && !shadow_only.is_empty() => { + "shadow_only" + } + ShadowDeliveryComparisonV1::Mismatch { .. } => "mismatch", + ShadowDeliveryComparisonV1::Ambiguous { .. } => "ambiguous", + }; + shadow_metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&[outcome]) + .inc(); + } + ShadowServiceEventV1::Input { kind, outcome } => { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&[kind, outcome]) + .inc(); + } + ShadowServiceEventV1::WalDurable { batches, records } => { + shadow_metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .inc_by(batches); + shadow_metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .inc_by(records); + } + ShadowServiceEventV1::Ready => { + if shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .get() + != 0 + { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(1); + } + } + ShadowServiceEventV1::Recovered { + batches, + discarded_tail_bytes, + } => { + shadow_metrics + .starfish_rbc_dag_shadow_wal_replayed_batches + .set(batches as i64); + shadow_metrics + .starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total + .inc_by(discarded_tail_bytes); + } + ShadowServiceEventV1::PendingRecovery(pending) => { + shadow_metrics + .starfish_rbc_dag_shadow_pending_recovery + .set(pending as i64); + } + ShadowServiceEventV1::Rejected { peer, error } => { + if peer.is_none() { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + tracing::warn!( + "Rejected non-authoritative RBC-DAG shadow input from {:?}: {}", + peer, + error + ); + } + } + } + }) + }); + // Start bridge task that forwards reconstructed transaction data to core let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); @@ -2174,6 +2537,8 @@ impl NetworkSyncer sf_event_task, rbc_event_task, rbc_service_task, + rbc_dag_shadow_event_task, + rbc_dag_shadow_service_task, cordial_knowledge_task, } } @@ -2219,6 +2584,11 @@ impl NetworkSyncer rbc_task.await.ok(); } let rbc_service_task = self.rbc_service_task; + if let Some(shadow_task) = self.rbc_dag_shadow_event_task { + shadow_task.abort(); + shadow_task.await.ok(); + } + let rbc_dag_shadow_service_task = self.rbc_dag_shadow_service_task; // Stop the cordial knowledge actor. self.cordial_knowledge_task.abort(); self.cordial_knowledge_task.await.ok(); @@ -2248,11 +2618,39 @@ impl NetworkSyncer // this FIFO barrier. Awaiting it keeps the runtime available to the // RBC actor while any earlier core action completes. let _ = inner.syncer.missing_parent_references().await; + let mut shadow_shutdown_timed_out = false; + if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { + match tokio::time::timeout(STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT, shadow.shutdown()) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!( + "Non-authoritative RBC-DAG shadow did not acknowledge shutdown: {error}" + ), + Err(_) => { + shadow_shutdown_timed_out = true; + tracing::warn!( + "Timed out stopping non-authoritative RBC-DAG shadow; detaching it from validator shutdown" + ); + if let Some(task) = rbc_dag_shadow_service_task.as_ref() { + task.abort(); + } + } + } + } let syncer = inner.syncer.stop(); if let Some(rbc_service_task) = rbc_service_task { rbc_service_task.abort(); rbc_service_task.await.ok(); } + if let Some(shadow_service_task) = rbc_dag_shadow_service_task { + match shadow_service_task.await { + Err(error) if !shadow_shutdown_timed_out => tracing::warn!( + "Non-authoritative RBC-DAG shadow supervisor failed during shutdown: {error}" + ), + _ => {} + } + } syncer } @@ -2364,6 +2762,7 @@ impl NetworkSyncer .await .ok()?; + let shadow_metrics = metrics.clone(); let mut handler = ConnectionHandler::new( &connection, universal_committer, @@ -2403,6 +2802,20 @@ impl NetworkSyncer ); } } + if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.peer_connected(peer_id) { + if shadow_transport_error_invalidates_comparison(&error) { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + tracing::warn!( + "Failed to notify RBC-DAG shadow that authority {} connected: {}", + peer_id, + error + ); + } + } if inner.dag_state.consensus_protocol.uses_bls() { for (round, signature) in inner.dag_state.precomputed_round_sigs() { @@ -2450,6 +2863,20 @@ impl NetworkSyncer ); } } + if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.peer_disconnected(peer_id) { + if shadow_transport_error_invalidates_comparison(&error) { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + tracing::warn!( + "Failed to notify RBC-DAG shadow that authority {} disconnected: {}", + peer_id, + error + ); + } + } inner.peer_senders.write().remove(&peer_id); inner.rbc_peer_senders.write().remove(&peer_id); if let Some(rbc_outbound_task) = rbc_outbound_task { diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 5ee30976..6531bd77 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -83,6 +83,24 @@ pub struct ShardPayload { pub shard: ProvableShard, } +/// Non-authoritative Starfish-RBC-DAG carrier used by the persisted shadow +/// runtime. Both byte strings use the versioned canonical codecs from +/// `starfish_rbc_dag`; the network envelope deliberately adds no second +/// identity or authentication scheme. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcDagShadowCarrier { + pub canonical_carrier: Vec, + pub authentication_sidecar: Vec, +} + +/// Content-only response for a phase-evidenced shadow carrier. Recovery can +/// satisfy READY/delivery, but it cannot grant optimistic admission or ECHO. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcDagShadowCarrierResponse { + pub reference: BlockReference, + pub canonical_carrier: Vec, +} + /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -190,6 +208,14 @@ pub enum NetworkMessage { /// Starfish-RBC: return canonical header content. The receiver recomputes /// and checks its content-addressed reference before accepting it. RbcHeaderResponse(RbcCanonicalHeader), + /// Starfish-RBC-DAG milestone-three shadow carrier. This path is + /// observational and never feeds the authoritative DAG or consensus. + RbcDagShadowCarrier(RbcDagShadowCarrier), + /// Request canonical content after embedded phase evidence arrives before + /// the corresponding shadow carrier. + RbcDagShadowCarrierRequest(BlockReference), + /// Return content only; the receiver recomputes and checks the reference. + RbcDagShadowCarrierResponse(RbcDagShadowCarrierResponse), } impl NetworkMessage { @@ -217,6 +243,9 @@ impl NetworkMessage { }, Self::RbcHeaderRequest(_) => "rbc_header_request", Self::RbcHeaderResponse(_) => "rbc_header_response", + Self::RbcDagShadowCarrier(_) => "rbc_dag_shadow_carrier", + Self::RbcDagShadowCarrierRequest(_) => "rbc_dag_shadow_carrier_request", + Self::RbcDagShadowCarrierResponse(_) => "rbc_dag_shadow_carrier_response", } } } @@ -544,7 +573,7 @@ impl Worker { // Spawn the first task for handling pings let writer_clone = Arc::clone(&writer); let bytes_sent_total_clone = bytes_sent_total.clone(); - let ping_task = tokio::spawn(async move { + let ping_task = async move { let mut ping_deadline = start + PING_INTERVAL; loop { tokio::time::sleep_until(ping_deadline).await; @@ -564,12 +593,12 @@ impl Worker { } bytes_sent_total_clone.inc_by(12); // ping is 12-byte sized } - }); + }; // Spawn the second task for handling pong responses let writer_clone = Arc::clone(&writer); let bytes_sent_total_clone = bytes_sent_total.clone(); - let pong_task = tokio::spawn(async move { + let pong_task = async move { while let Some(ping) = pong_receiver.recv().await { if ping == 0 { tracing::warn!("Invalid ping: {ping}"); @@ -616,7 +645,7 @@ impl Worker { // Yield to ensure responsiveness tokio::task::yield_now().await; } - }); + }; // Spawn the third task(s) for handling message sending. // @@ -625,7 +654,7 @@ impl Worker { // backpressure encoding and starve catch-up for late joiners under // latency simulation. if connection_latency == 0.0 { - let message_task = tokio::spawn(async move { + let message_task = async move { while let Some(message) = receiver.recv().await { let request_type = message.request_type(); let serialized = bincode::serialize(&message).expect("Serialization failed"); @@ -661,10 +690,10 @@ impl Worker { } } } - }); + }; // Wait for all tasks to complete. - let _ = tokio::try_join!(ping_task, pong_task, message_task); + let _ = tokio::join!(ping_task, pong_task, message_task); return Ok(()); } @@ -672,7 +701,7 @@ impl Worker { // behavior by sleeping inside per-message tasks, but keep concurrency // bounded to avoid unbounded task buildup under heavy load. const MAX_IN_FLIGHT: usize = NETWORK_MESSAGE_CHANNEL_CAPACITY * 64; - let message_task = tokio::spawn(async move { + let message_task = async move { let mut join_set = tokio::task::JoinSet::new(); while let Some(message) = receiver.recv().await { @@ -731,10 +760,10 @@ impl Worker { tracing::error!("An inner task failed: {e:?}"); } } - }); + }; // Wait for all tasks to complete. - let _ = tokio::try_join!(ping_task, pong_task, message_task); + let _ = tokio::join!(ping_task, pong_task, message_task); Ok(()) } @@ -962,12 +991,119 @@ fn decode_ping(message: &[u8]) -> i64 { #[cfg(test)] mod tests { + use prometheus::Registry; + use super::*; use crate::{ + committee::Committee, crypto::{MacTag, TransactionsCommitment, dummy_signer}, starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, }; + const NETWORK_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(6); + + async fn connected_pair( + addresses: &[SocketAddr; 2], + parameters: &NodeParameters, + ) -> (Network, Network, Connection, Connection) { + let committee = Committee::new_for_benchmarks(2); + let metrics_0 = Metrics::new(&Registry::new(), Some(&committee), None, None).0; + let metrics_1 = Metrics::new(&Registry::new(), Some(&committee), None, None).0; + let mut network_0 = + Network::from_socket_addresses(addresses, 0, addresses[0], metrics_0, parameters).await; + let mut network_1 = + Network::from_socket_addresses(addresses, 1, addresses[1], metrics_1, parameters).await; + + let (connection_0, connection_1) = tokio::time::timeout(NETWORK_LIFECYCLE_TIMEOUT, async { + tokio::join!( + network_0.connection_receiver().recv(), + network_1.connection_receiver().recv(), + ) + }) + .await + .expect("two-node network did not connect before the lifecycle timeout"); + let connection_0 = connection_0.expect("authority 0 connection channel closed"); + let connection_1 = connection_1.expect("authority 1 connection channel closed"); + assert_eq!(connection_0.peer_id, 1); + assert_eq!(connection_1.peer_id, 0); + (network_0, network_1, connection_0, connection_1) + } + + async fn assert_bidirectional_round_trip( + connection_0: &mut Connection, + connection_1: &mut Connection, + marker: RoundNumber, + ) { + connection_0 + .sender + .send(NetworkMessage::SubscribeBroadcastRequest(marker)) + .await + .unwrap(); + connection_1 + .sender + .send(NetworkMessage::SubscribeBroadcastRequest(marker + 1)) + .await + .unwrap(); + + let (received_by_0, received_by_1) = + tokio::time::timeout(NETWORK_LIFECYCLE_TIMEOUT, async { + tokio::join!(connection_0.receiver.recv(), connection_1.receiver.recv()) + }) + .await + .expect("two-node network did not exchange messages before the lifecycle timeout"); + assert!(matches!( + received_by_0, + Some(NetworkMessage::SubscribeBroadcastRequest(round)) if round == marker + 1 + )); + assert!(matches!( + received_by_1, + Some(NetworkMessage::SubscribeBroadcastRequest(round)) if round == marker + )); + } + + async fn abort_network_pair(network_0: Network, network_1: Network) { + // Match production shutdown: abort both listeners together. Awaiting + // the server tasks makes listener release deterministic for this test; + // dropping their worker senders must then cancel every scoped stream + // future and its OwnedWriteHalf. + network_0.abort_server(); + network_1.abort_server(); + let Network { + connection_receiver: connection_receiver_0, + server_task: server_task_0, + } = network_0; + let Network { + connection_receiver: connection_receiver_1, + server_task: server_task_1, + } = network_1; + drop(connection_receiver_0); + drop(connection_receiver_1); + let (result_0, result_1) = tokio::join!(server_task_0, server_task_1); + assert!(result_0.is_err_and(|error| error.is_cancelled())); + assert!(result_1.is_err_and(|error| error.is_cancelled())); + } + + async fn same_port_rebind_case(addresses: [SocketAddr; 2], latency_ms: Option) { + let parameters = NodeParameters { + mimic_latency: false, + uniform_latency_ms: latency_ms, + ..NodeParameters::default() + }; + + for cycle in 0..2 { + let (network_0, network_1, mut connection_0, mut connection_1) = + connected_pair(&addresses, ¶meters).await; + assert_bidirectional_round_trip(&mut connection_0, &mut connection_1, 10 + cycle).await; + + // NetworkSyncer drops its connection tasks before aborting the + // listener. Reproduce that ordering, then immediately construct + // the next cycle on the identical listener and active-bind ports. + drop(connection_0); + drop(connection_1); + abort_network_pair(network_0, network_1).await; + } + } + fn variant_index(message: &NetworkMessage) -> u32 { let bytes = bincode::serialize(message).unwrap(); u32::from_le_bytes(bytes[..4].try_into().unwrap()) @@ -1006,12 +1142,25 @@ mod tests { )); let request = NetworkMessage::RbcHeaderRequest(block_ref); let response = NetworkMessage::RbcHeaderResponse(header); + let shadow = NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { + canonical_carrier: vec![0xA3, 0xA4], + authentication_sidecar: vec![0xA5], + }); + let shadow_request = NetworkMessage::RbcDagShadowCarrierRequest(block_ref); + let shadow_response = + NetworkMessage::RbcDagShadowCarrierResponse(RbcDagShadowCarrierResponse { + reference: block_ref, + canonical_carrier: vec![0xA6, 0xA7], + }); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), (phase, 12, "rbc_ready"), (request, 13, "rbc_header_request"), (response, 14, "rbc_header_response"), + (shadow, 15, "rbc_dag_shadow_carrier"), + (shadow_request, 16, "rbc_dag_shadow_carrier_request"), + (shadow_response, 17, "rbc_dag_shadow_carrier_response"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); @@ -1021,4 +1170,29 @@ mod tests { assert_eq!(variant_index(&decoded), expected_index); } } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn scoped_connection_tasks_allow_immediate_same_port_rebind() { + // Active sockets bind to listener_port * 10. Keep those derived ports + // below the usual Linux ephemeral range to avoid unrelated allocation + // races while staying above the repository's validator-test fixtures. + same_port_rebind_case( + [ + SocketAddr::from(([127, 0, 0, 1], 3_200)), + SocketAddr::from(([127, 0, 0, 1], 3_201)), + ], + None, + ) + .await; + // Any nonzero configured latency selects the JoinSet-backed writer + // branch, so this also covers cancellation of its in-flight tasks. + same_port_rebind_case( + [ + SocketAddr::from(([127, 0, 0, 1], 3_220)), + SocketAddr::from(([127, 0, 0, 1], 3_221)), + ], + Some(5.0), + ) + .await; + } } diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index fa7287a6..24812f40 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -8,7 +8,7 @@ //! decoding. Callers validate canonical bytes before journaling them; the //! reducer pins the exact byte strings and rejects any later alternative. -use std::{collections::BTreeMap, error::Error, fmt}; +use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; use crate::types::{AuthorityIndex, BlockReference, RoundNumber}; @@ -1010,6 +1010,10 @@ pub enum JournalErrorV1 { outer: BlockReference, index: usize, }, + StaleValidatedBatch { + expected_events: usize, + actual_events: usize, + }, } impl fmt::Display for JournalErrorV1 { @@ -1026,7 +1030,17 @@ pub struct WriteAheadJournalV1 { context: RbcDagContextV1, own_authority: AuthorityIndex, durable_events: Vec, - snapshot: JournalSnapshotV1, + snapshot: Arc, +} + +/// A transition checked against one exact journal prefix. Keeping only the +/// newly appended events avoids cloning the complete durable history on every +/// live shadow input. +pub(crate) struct ValidatedJournalBatchV1 { + base_event_count: usize, + base_snapshot: Arc, + events: Vec, + snapshot: Arc, } impl WriteAheadJournalV1 { @@ -1035,7 +1049,7 @@ impl WriteAheadJournalV1 { context, own_authority, durable_events: Vec::new(), - snapshot: JournalSnapshotV1::new(context, own_authority), + snapshot: Arc::new(JournalSnapshotV1::new(context, own_authority)), } } @@ -1043,10 +1057,43 @@ impl WriteAheadJournalV1 { /// durable and visible. A real backend maps this boundary to its durable /// transaction commit. pub fn append(&mut self, event: JournalEventV1) -> Result<(), JournalErrorV1> { - let mut next = self.snapshot.clone(); + let mut next = self.snapshot.as_ref().clone(); next.apply(&event)?; self.durable_events.push(event); - self.snapshot = next; + self.snapshot = Arc::new(next); + Ok(()) + } + + pub(crate) fn validate_batch( + &self, + events: Vec, + ) -> Result { + let mut snapshot = self.snapshot.as_ref().clone(); + for event in &events { + snapshot.apply(event)?; + } + Ok(ValidatedJournalBatchV1 { + base_event_count: self.durable_events.len(), + base_snapshot: Arc::clone(&self.snapshot), + events, + snapshot: Arc::new(snapshot), + }) + } + + pub(crate) fn commit_validated_batch( + &mut self, + batch: ValidatedJournalBatchV1, + ) -> Result<(), JournalErrorV1> { + if self.durable_events.len() != batch.base_event_count + || !Arc::ptr_eq(&self.snapshot, &batch.base_snapshot) + { + return Err(JournalErrorV1::StaleValidatedBatch { + expected_events: batch.base_event_count, + actual_events: self.durable_events.len(), + }); + } + self.durable_events.extend(batch.events); + self.snapshot = batch.snapshot; Ok(()) } @@ -1070,7 +1117,7 @@ impl WriteAheadJournalV1 { } pub fn snapshot(&self) -> &JournalSnapshotV1 { - &self.snapshot + self.snapshot.as_ref() } /// Rebuild volatile state in exact durable order. @@ -1095,7 +1142,7 @@ impl WriteAheadJournalV1 { context, own_authority, durable_events, - snapshot, + snapshot: Arc::new(snapshot), }) } } @@ -1371,6 +1418,121 @@ mod tests { assert!(assertion(after.snapshot())); } + #[test] + fn validated_batch_matches_sequential_append_and_restart() { + let mut batched = journal(); + let mut sequential = batched.clone(); + let own_candidate = candidate(1, 1, 0x0A, Vec::new(), None); + let events = vec![ + outbound_content_event(&batched, &own_candidate), + fix_event(&batched, own_candidate.reference()), + ]; + + let batch = batched.validate_batch(events.clone()).unwrap(); + batched.commit_validated_batch(batch).unwrap(); + for event in events { + sequential.append(event).unwrap(); + } + + assert_eq!(batched.durable_events(), sequential.durable_events()); + assert_eq!(batched.snapshot(), sequential.snapshot()); + assert_eq!(batched.restart().unwrap(), sequential.restart().unwrap()); + } + + #[test] + fn failed_batch_validation_leaves_the_journal_unchanged() { + let journal = journal(); + let before = journal.clone(); + let own = candidate(1, 1, 0x0B, Vec::new(), None).reference(); + + let result = journal.validate_batch(vec![fix_event(&journal, own)]); + + assert!(matches!( + result, + Err(JournalErrorV1::OutboundContentNotPersisted(reference)) if reference == own + )); + assert_eq!(journal, before); + } + + #[test] + fn validated_batch_rejects_an_intervening_append_without_mutation() { + let mut journal = journal(); + let planned = candidate(0, 1, 0x0C, Vec::new(), None); + let intervening = candidate(2, 1, 0x0D, Vec::new(), None); + let batch = journal + .validate_batch(vec![JournalEventV1::RetainCandidateContent { + context: journal.context, + candidate: planned, + }]) + .unwrap(); + journal + .append(JournalEventV1::RetainCandidateContent { + context: journal.context, + candidate: intervening, + }) + .unwrap(); + let after_intervening = journal.clone(); + + assert_eq!( + journal.commit_validated_batch(batch).unwrap_err(), + JournalErrorV1::StaleValidatedBatch { + expected_events: 0, + actual_events: 1, + } + ); + assert_eq!(journal, after_intervening); + assert_eq!( + journal.restart().unwrap(), + after_intervening.restart().unwrap() + ); + } + + #[test] + fn validated_batch_rejects_a_divergent_equal_length_journal() { + let mut source = journal(); + let mut divergent = source.clone(); + let source_candidate = candidate(0, 1, 0x0E, Vec::new(), None); + let divergent_candidate = candidate(2, 1, 0x0F, Vec::new(), None); + source + .append(JournalEventV1::RetainCandidateContent { + context: source.context, + candidate: source_candidate.clone(), + }) + .unwrap(); + divergent + .append(JournalEventV1::RetainCandidateContent { + context: divergent.context, + candidate: divergent_candidate, + }) + .unwrap(); + assert_eq!( + source.durable_events().len(), + divergent.durable_events().len() + ); + assert_ne!(source.snapshot(), divergent.snapshot()); + + let batch = source + .validate_batch(vec![JournalEventV1::LockReady { + context: source.context, + target: source_candidate.reference(), + }]) + .unwrap(); + let before_commit = divergent.clone(); + + assert_eq!( + divergent.commit_validated_batch(batch).unwrap_err(), + JournalErrorV1::StaleValidatedBatch { + expected_events: 1, + actual_events: 1, + } + ); + assert_eq!(divergent, before_commit); + assert_eq!( + divergent.restart().unwrap(), + before_commit.restart().unwrap() + ); + } + #[test] fn authenticated_ingress_sequence_and_bytes_survive_restart_in_order() { let mut journal = journal(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs index 2a3afd17..31f82af0 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -4,12 +4,14 @@ //! Canonical carrier types for the experimental embedded-RBC Starfish DAG. //! //! This module is deliberately independent from the implemented direct-message -//! `starfish_rbc` protocol. Runtime and consensus integration are later -//! milestones. +//! `starfish_rbc` protocol. An opt-in persisted shadow adapter consumes these +//! types without influencing consensus; authoritative integration remains a +//! later milestone. pub mod journal; pub mod model; pub mod projection; +pub mod storage; use std::{ collections::{BTreeSet, HashSet}, @@ -67,6 +69,11 @@ const CARRIER_AUTHENTICATION_KIND: u8 = 0; const AUTHENTICATION_BASE_SIZE: usize = 123; const AUTHENTICATION_MAC_SIZE: usize = AUTHENTICATION_BASE_SIZE + 2; +#[cfg(test)] +std::thread_local! { + static COMMITTEE_ID_DERIVATIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum RbcPhaseStatementV1 { Echo { target: BlockReference }, @@ -403,19 +410,59 @@ pub struct CandidateCarrierV1 { } impl CandidateCarrierV1 { + /// Convenience constructor for tests and one-shot callers. + /// + /// Runtime code should build one [`RbcDagCommitteeContextV1`] and call + /// [`Self::try_new_with_committee`] so the committee's complete public-key + /// transcript is not rehashed for every carrier. pub fn try_new(args: CarrierHeaderV1Args, committee: &Committee) -> Result { - Self::try_from_header(CarrierHeaderV1::from_args(args), committee, None) + Self::try_from_header_internal(CarrierHeaderV1::from_args(args), committee, None, None) + } + + pub fn try_new_with_committee( + args: CarrierHeaderV1Args, + committee: &RbcDagCommitteeContextV1, + ) -> Result { + Self::try_from_header_internal( + CarrierHeaderV1::from_args(args), + committee.committee(), + Some(committee.committee_id()), + None, + ) } + /// Convenience constructor for tests and one-shot callers. Runtime code + /// should prefer [`Self::try_from_header_with_committee`]. pub fn try_from_header( + header: CarrierHeaderV1, + committee: &Committee, + expected_reference: Option, + ) -> Result { + Self::try_from_header_internal(header, committee, None, expected_reference) + } + + pub fn try_from_header_with_committee( + header: CarrierHeaderV1, + committee: &RbcDagCommitteeContextV1, + expected_reference: Option, + ) -> Result { + Self::try_from_header_internal( + header, + committee.committee(), + Some(committee.committee_id()), + expected_reference, + ) + } + + fn try_from_header_internal( mut header: CarrierHeaderV1, committee: &Committee, + cached_committee_id: Option, expected_reference: Option, ) -> Result { normalize_acknowledgments(&mut header)?; - validate_outer_header(&header, committee)?; + validate_outer_header(&header, committee, cached_committee_id.is_some())?; let reference = carrier_reference(&header)?; - let committee_id = RbcDagCommitteeId::derive(committee)?; if let Some(expected) = expected_reference { if expected != reference { return Err(RbcDagError::ReferenceMismatch { @@ -424,6 +471,10 @@ impl CandidateCarrierV1 { }); } } + let committee_id = match cached_committee_id { + Some(committee_id) => committee_id, + None => RbcDagCommitteeId::derive(committee)?, + }; Ok(Self { header: Arc::new(header), reference, @@ -437,7 +488,26 @@ impl CandidateCarrierV1 { expected_reference: Option, ) -> Result { let header = decode_header(bytes, AckEncoding::Expanded)?; - let candidate = Self::try_from_header(header, committee, expected_reference)?; + let candidate = + Self::try_from_header_internal(header, committee, None, expected_reference)?; + if candidate.canonical_content_bytes()?.as_slice() != bytes { + return Err(RbcDagError::NonCanonicalAcknowledgments); + } + Ok(candidate) + } + + pub fn decode_content_with_committee( + bytes: &[u8], + committee: &RbcDagCommitteeContextV1, + expected_reference: Option, + ) -> Result { + let header = decode_header(bytes, AckEncoding::Expanded)?; + let candidate = Self::try_from_header_internal( + header, + committee.committee(), + Some(committee.committee_id()), + expected_reference, + )?; if candidate.canonical_content_bytes()?.as_slice() != bytes { return Err(RbcDagError::NonCanonicalAcknowledgments); } @@ -450,7 +520,21 @@ impl CandidateCarrierV1 { expected_reference: Option, ) -> Result { let header = decode_header(bytes, AckEncoding::Compressed)?; - Self::try_from_header(header, committee, expected_reference) + Self::try_from_header_internal(header, committee, None, expected_reference) + } + + pub fn decode_wire_with_committee( + bytes: &[u8], + committee: &RbcDagCommitteeContextV1, + expected_reference: Option, + ) -> Result { + let header = decode_header(bytes, AckEncoding::Compressed)?; + Self::try_from_header_internal( + header, + committee.committee(), + Some(committee.committee_id()), + expected_reference, + ) } pub fn header(&self) -> &CarrierHeaderV1 { @@ -479,6 +563,24 @@ impl CandidateCarrierV1 { ) -> Result, RbcDagProjectionError> { let committee_id = RbcDagCommitteeId::derive(committee) .map_err(|_| RbcDagProjectionError::CommitteeMismatch)?; + self.validate_consensus_vertex_with_validated_committee(committee, committee_id) + } + + pub fn validate_consensus_vertex_with_committee( + &self, + committee: &RbcDagCommitteeContextV1, + ) -> Result, RbcDagProjectionError> { + self.validate_consensus_vertex_with_validated_committee( + committee.committee(), + committee.committee_id(), + ) + } + + fn validate_consensus_vertex_with_validated_committee( + &self, + committee: &Committee, + committee_id: RbcDagCommitteeId, + ) -> Result, RbcDagProjectionError> { if committee_id != self.committee_id { return Err(RbcDagProjectionError::CommitteeMismatch); } @@ -587,8 +689,24 @@ impl CarrierAuthenticationV1 { bytes } + /// Convenience decoder for one-shot callers. Runtime code should prefer + /// [`Self::decode_wire_with_committee`]. pub fn decode_wire(bytes: &[u8], committee: &Committee) -> Result { validate_committee(committee)?; + Self::decode_wire_with_validated_committee(bytes, committee) + } + + pub fn decode_wire_with_committee( + bytes: &[u8], + committee: &RbcDagCommitteeContextV1, + ) -> Result { + Self::decode_wire_with_validated_committee(bytes, committee.committee()) + } + + fn decode_wire_with_validated_committee( + bytes: &[u8], + committee: &Committee, + ) -> Result { let mut decoder = Decoder::new(bytes); decoder.expect_marker(CONTENT_FORMAT_FIELD)?; let version = decoder.read_u8()?; @@ -687,6 +805,8 @@ pub struct RbcDagCommitteeId([u8; COMMITTEE_ID_SIZE]); impl RbcDagCommitteeId { pub fn derive(committee: &Committee) -> Result { validate_committee(committee)?; + #[cfg(test)] + COMMITTEE_ID_DERIVATIONS.with(|count| count.set(count.get().saturating_add(1))); let committee_size = u16::try_from(committee.len()) .map_err(|_| RbcDagError::InvalidCommittee("committee too large"))?; let info_length = u16::try_from(committee.info_length()) @@ -736,6 +856,51 @@ impl fmt::Debug for RbcDagCommitteeId { } } +/// Validated, reusable committee capability for the Starfish-RBC-DAG hot +/// path. +/// +/// Construction validates the committee and hashes its complete key +/// transcript exactly once. Candidate decoding, authentication, and +/// projection APIs that accept this capability perform only constant-time ID +/// comparisons before using the retained committee. +#[derive(Clone)] +pub struct RbcDagCommitteeContextV1 { + committee: Arc, + committee_id: RbcDagCommitteeId, +} + +impl RbcDagCommitteeContextV1 { + pub fn new(committee: Arc) -> Result { + let committee_id = RbcDagCommitteeId::derive(&committee)?; + Ok(Self { + committee, + committee_id, + }) + } + + pub fn committee(&self) -> &Committee { + &self.committee + } + + pub fn committee_arc(&self) -> Arc { + Arc::clone(&self.committee) + } + + pub fn committee_id(&self) -> RbcDagCommitteeId { + self.committee_id + } +} + +impl fmt::Debug for RbcDagCommitteeContextV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RbcDagCommitteeContextV1") + .field("committee_id", &self.committee_id) + .field("committee_size", &self.committee.len()) + .finish() + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RbcDagContextV1 { protocol_instance: RbcDagProtocolInstanceId, @@ -744,6 +909,8 @@ pub struct RbcDagContextV1 { } impl RbcDagContextV1 { + /// Convenience constructor for one-shot callers. Runtime code should + /// prefer [`Self::new_with_committee`]. pub fn new( protocol_instance: RbcDagProtocolInstanceId, committee: &Committee, @@ -756,6 +923,18 @@ impl RbcDagContextV1 { }) } + pub fn new_with_committee( + protocol_instance: RbcDagProtocolInstanceId, + committee: &RbcDagCommitteeContextV1, + authentication_scheme: BlockAuthenticationScheme, + ) -> Self { + Self { + protocol_instance, + committee_id: committee.committee_id(), + authentication_scheme, + } + } + pub fn protocol_instance(&self) -> RbcDagProtocolInstanceId { self.protocol_instance } @@ -768,13 +947,40 @@ impl RbcDagContextV1 { self.authentication_scheme } + /// Convenience authorizer for one-shot callers. Runtime code should + /// prefer [`Self::authenticate_with_committee`]. pub fn authenticate( &self, candidate: &CandidateCarrierV1, committee: &Committee, authorizer: CarrierAuthorizerV1<'_>, ) -> Result { - self.ensure_committee(committee)?; + let committee_id = RbcDagCommitteeId::derive(committee)?; + self.authenticate_with_validated_committee(candidate, committee, committee_id, authorizer) + } + + pub fn authenticate_with_committee( + &self, + candidate: &CandidateCarrierV1, + committee: &RbcDagCommitteeContextV1, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + self.authenticate_with_validated_committee( + candidate, + committee.committee(), + committee.committee_id(), + authorizer, + ) + } + + fn authenticate_with_validated_committee( + &self, + candidate: &CandidateCarrierV1, + committee: &Committee, + committee_id: RbcDagCommitteeId, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + self.ensure_committee_id(committee_id)?; self.ensure_candidate(candidate)?; if authorizer.scheme() != self.authentication_scheme { return Err(RbcDagError::AuthenticationSchemeMismatch); @@ -850,6 +1056,9 @@ impl RbcDagContextV1 { /// The returned capability has private fields so persistence and network /// adapters cannot substitute a freely constructed, same-scheme sidecar /// for the one produced by the configured authorizer. + /// + /// Convenience local authorizer for one-shot callers. Runtime code should + /// prefer [`Self::authenticate_local_with_committee`]. pub fn authenticate_local( &self, candidate: CandidateCarrierV1, @@ -864,6 +1073,175 @@ impl RbcDagContextV1 { }) } + pub fn authenticate_local_with_committee( + &self, + candidate: CandidateCarrierV1, + committee: &RbcDagCommitteeContextV1, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + let authentication = self.authenticate_with_committee(&candidate, committee, authorizer)?; + Ok(LocallyAuthenticatedCarrierV1 { + candidate, + authentication, + context: *self, + }) + } + + /// Recover the opaque local-authentication capability from an exact + /// persisted sidecar without regenerating it. + /// + /// Signature modes verify the persisted public proof and the configured + /// local signer's public key. MAC mode verifies every ordered vector entry + /// with the configured outbound keyring; checking only this node's entry + /// would not prove that the locally exposed full vector was generated + /// correctly. + /// + /// Convenience recovery verifier for one-shot callers. Runtime code + /// should prefer [`Self::verify_local_authentication_with_committee`]. + pub fn verify_local_authentication( + &self, + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + committee: &Committee, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + let committee_id = RbcDagCommitteeId::derive(committee)?; + self.verify_local_authentication_with_validated_committee( + candidate, + authentication, + committee, + committee_id, + authorizer, + ) + } + + pub fn verify_local_authentication_with_committee( + &self, + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + committee: &RbcDagCommitteeContextV1, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + self.verify_local_authentication_with_validated_committee( + candidate, + authentication, + committee.committee(), + committee.committee_id(), + authorizer, + ) + } + + fn verify_local_authentication_with_validated_committee( + &self, + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + committee: &Committee, + committee_id: RbcDagCommitteeId, + authorizer: CarrierAuthorizerV1<'_>, + ) -> Result { + self.ensure_committee_id(committee_id)?; + self.ensure_candidate(&candidate)?; + if authentication.scheme() != self.authentication_scheme + || authorizer.scheme() != self.authentication_scheme + { + return Err(RbcDagError::AuthenticationSchemeMismatch); + } + let reference = candidate.reference; + if authorizer.authority() != reference.authority { + return Err(RbcDagError::AuthorizerAuthorityMismatch { + expected: reference.authority, + actual: authorizer.authority(), + }); + } + + match (authorizer, &authentication) { + ( + CarrierAuthorizerV1::Ed25519 { signer, .. }, + CarrierAuthenticationV1::Ed25519(signature), + ) => { + let expected = committee + .get_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + if &signer.public_key() != expected { + return Err(RbcDagError::AuthorizerKeyMismatch); + } + expected + .verify_digest_signature( + &self.public_authentication_digest(reference), + signature, + ) + .map_err(|_| RbcDagError::InvalidAuthentication)?; + } + ( + CarrierAuthorizerV1::MlDsa44 { signer, .. }, + CarrierAuthenticationV1::MlDsa44(signature), + ) => { + let expected = committee + .get_ml_dsa_44_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + if &signer.public_key() != expected { + return Err(RbcDagError::AuthorizerKeyMismatch); + } + let digest = BlockDigest::from(self.public_authentication_digest(reference)); + expected + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcDagError::InvalidAuthentication)?; + } + ( + CarrierAuthorizerV1::MlDsa65 { signer, .. }, + CarrierAuthenticationV1::MlDsa65(signature), + ) => { + let expected = committee + .get_ml_dsa_65_public_key(reference.authority) + .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; + if &signer.public_key() != expected { + return Err(RbcDagError::AuthorizerKeyMismatch); + } + let digest = BlockDigest::from(self.public_authentication_digest(reference)); + expected + .verify_digest_signature(&digest, signature) + .map_err(|_| RbcDagError::InvalidAuthentication)?; + } + ( + CarrierAuthorizerV1::MacVector { keys, .. }, + CarrierAuthenticationV1::MacVector(vector), + ) => { + let expected_length = committee.len() * MAC_TAG_SIZE; + if vector.as_bytes().len() != expected_length { + return Err(RbcDagError::InvalidMacVectorLength { + expected: expected_length, + actual: vector.as_bytes().len(), + }); + } + if keys.len() != committee.len() { + return Err(RbcDagError::InvalidKeyringLength { + expected: committee.len(), + actual: keys.len(), + }); + } + for recipient in committee.authorities() { + let expected = keys[recipient as usize] + .compute_rbc_tag(&self.mac_authentication_statement(reference, recipient)); + let actual = vector + .tag(recipient) + .ok_or(RbcDagError::InvalidAuthentication)?; + if actual != expected { + return Err(RbcDagError::InvalidAuthentication); + } + } + } + _ => return Err(RbcDagError::AuthenticationSchemeMismatch), + } + + Ok(LocallyAuthenticatedCarrierV1 { + candidate, + authentication, + context: *self, + }) + } + + /// Convenience inbound verifier for one-shot callers. Runtime code should + /// prefer [`Self::verify_authentication_with_committee`]. pub fn verify_authentication( &self, candidate: CandidateCarrierV1, @@ -872,7 +1250,45 @@ impl RbcDagContextV1 { committee: &Committee, mac_keys: &[MacKey], ) -> Result { - self.ensure_committee(committee)?; + let committee_id = RbcDagCommitteeId::derive(committee)?; + self.verify_authentication_with_validated_committee( + candidate, + authentication, + receiver, + committee, + committee_id, + mac_keys, + ) + } + + pub fn verify_authentication_with_committee( + &self, + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + receiver: AuthorityIndex, + committee: &RbcDagCommitteeContextV1, + mac_keys: &[MacKey], + ) -> Result { + self.verify_authentication_with_validated_committee( + candidate, + authentication, + receiver, + committee.committee(), + committee.committee_id(), + mac_keys, + ) + } + + fn verify_authentication_with_validated_committee( + &self, + candidate: CandidateCarrierV1, + authentication: CarrierAuthenticationV1, + receiver: AuthorityIndex, + committee: &Committee, + committee_id: RbcDagCommitteeId, + mac_keys: &[MacKey], + ) -> Result { + self.ensure_committee_id(committee_id)?; self.ensure_candidate(&candidate)?; if !committee.known_authority(receiver) { return Err(RbcDagError::UnknownAuthority(receiver)); @@ -969,8 +1385,7 @@ impl RbcDagContextV1 { blake3::hash(&self.public_authentication_statement(reference)).into() } - fn ensure_committee(&self, committee: &Committee) -> Result<(), RbcDagError> { - let actual = RbcDagCommitteeId::derive(committee)?; + fn ensure_committee_id(&self, actual: RbcDagCommitteeId) -> Result<(), RbcDagError> { if actual != self.committee_id { return Err(RbcDagError::CommitteeIdMismatch); } @@ -1169,8 +1584,11 @@ impl Error for RbcDagError {} fn validate_outer_header( header: &CarrierHeaderV1, committee: &Committee, + committee_is_validated: bool, ) -> Result<(), RbcDagError> { - validate_committee(committee)?; + if !committee_is_validated { + validate_committee(committee)?; + } if header.carrier_round == 0 { return Err(RbcDagError::GenesisCarrier); } @@ -2478,6 +2896,361 @@ mod tests { )); } + #[test] + fn cached_committee_context_hashes_the_key_transcript_once_across_hot_paths() { + COMMITTEE_ID_DERIVATIONS.with(|count| count.set(0)); + + let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + assert_eq!(COMMITTEE_ID_DERIVATIONS.with(std::cell::Cell::get), 1); + + let candidate = + CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) + .unwrap(); + let content = candidate.canonical_content_bytes().unwrap(); + let wire = candidate.canonical_wire_bytes().unwrap(); + let decoded_content = CandidateCarrierV1::decode_content_with_committee( + &content, + &committee_context, + Some(candidate.reference()), + ) + .unwrap(); + let decoded_wire = CandidateCarrierV1::decode_wire_with_committee( + &wire, + &committee_context, + Some(candidate.reference()), + ) + .unwrap(); + assert_eq!(decoded_content, candidate); + assert_eq!(decoded_wire, candidate); + + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(), + &committee_context, + BlockAuthenticationScheme::MacVector, + ); + let keyrings = mac_keyrings_for_test(committee.len()); + let authentication = context + .authenticate_with_committee( + &candidate, + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: candidate.header().author(), + keys: &keyrings[candidate.header().author() as usize], + }, + ) + .unwrap(); + let authentication_wire = authentication.canonical_wire_bytes(); + let decoded_authentication = CarrierAuthenticationV1::decode_wire_with_committee( + &authentication_wire, + &committee_context, + ) + .unwrap(); + context + .verify_authentication_with_committee( + candidate.clone(), + decoded_authentication, + 1, + &committee_context, + &keyrings[1], + ) + .unwrap(); + context + .verify_local_authentication_with_committee( + candidate.clone(), + authentication, + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: candidate.header().author(), + keys: &keyrings[candidate.header().author() as usize], + }, + ) + .unwrap(); + context + .authenticate_local_with_committee( + candidate.clone(), + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: candidate.header().author(), + keys: &keyrings[candidate.header().author() as usize], + }, + ) + .unwrap(); + candidate + .validate_consensus_vertex_with_committee(&committee_context) + .unwrap(); + + let mut projection = + projection::CertifiedProjectionModel::from_committee_context(committee_context.clone()); + projection.stage_carrier(candidate.clone()).unwrap(); + assert!(matches!( + projection.try_project(candidate.reference()), + Err(projection::CertifiedProjectionError::CarrierNotDelivered(reference)) + if reference == candidate.reference() + )); + + assert_eq!(COMMITTEE_ID_DERIVATIONS.with(std::cell::Cell::get), 1); + } + + #[test] + fn cached_committee_context_rejects_cross_committee_hot_path_use() { + let committee = Committee::new_test(vec![1; 4]); + let other_committee = Committee::new_test(vec![1, 1, 1, 2]); + let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let other_context = RbcDagCommitteeContextV1::new(Arc::clone(&other_committee)).unwrap(); + let candidate = + CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) + .unwrap(); + let protocol_context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xB7; 32]).unwrap(), + &committee_context, + BlockAuthenticationScheme::MacVector, + ); + let keyrings = mac_keyrings_for_test(committee.len()); + let authentication = protocol_context + .authenticate_with_committee( + &candidate, + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: candidate.header().author(), + keys: &keyrings[candidate.header().author() as usize], + }, + ) + .unwrap(); + + assert!(matches!( + protocol_context.authenticate_with_committee( + &candidate, + &other_context, + CarrierAuthorizerV1::MacVector { + authority: candidate.header().author(), + keys: &keyrings[candidate.header().author() as usize], + }, + ), + Err(RbcDagError::CommitteeIdMismatch) + )); + assert!(matches!( + protocol_context.verify_authentication_with_committee( + candidate.clone(), + authentication.clone(), + 1, + &other_context, + &keyrings[1], + ), + Err(RbcDagError::CommitteeIdMismatch) + )); + assert!(matches!( + protocol_context.verify_local_authentication_with_committee( + candidate.clone(), + authentication, + &other_context, + CarrierAuthorizerV1::MacVector { + authority: candidate.header().author(), + keys: &keyrings[candidate.header().author() as usize], + }, + ), + Err(RbcDagError::CommitteeIdMismatch) + )); + assert!(matches!( + candidate.validate_consensus_vertex_with_committee(&other_context), + Err(RbcDagProjectionError::CommitteeMismatch) + )); + + let mut projection = + projection::CertifiedProjectionModel::from_committee_context(other_context); + assert_eq!( + projection.stage_carrier(candidate), + Err(projection::CertifiedProjectionError::CommitteeMismatch) + ); + } + + #[test] + fn persisted_ml_dsa_sidecar_recovers_exact_local_capability_and_rejects_tampering() { + let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let candidate = + CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) + .unwrap(); + let instance = RbcDagProtocolInstanceId::new([0xC8; 32]).unwrap(); + let context = RbcDagContextV1::new_with_committee( + instance, + &committee_context, + BlockAuthenticationScheme::MlDsa65, + ); + let signer = dummy_ml_dsa_65_signer(); + let authentication = context + .authenticate_with_committee( + &candidate, + &committee_context, + CarrierAuthorizerV1::MlDsa65 { + authority: candidate.header().author(), + signer: &signer, + }, + ) + .unwrap(); + let persisted_wire = authentication.canonical_wire_bytes(); + let persisted_authentication = CarrierAuthenticationV1::decode_wire_with_committee( + &persisted_wire, + &committee_context, + ) + .unwrap(); + let recovered = context + .verify_local_authentication_with_committee( + candidate.clone(), + persisted_authentication, + &committee_context, + CarrierAuthorizerV1::MlDsa65 { + authority: candidate.header().author(), + signer: &signer, + }, + ) + .unwrap(); + assert_eq!(recovered.authentication(), &authentication); + assert_eq!( + recovered.authentication().canonical_wire_bytes(), + persisted_wire + ); + + let CarrierAuthenticationV1::MlDsa65(signature) = &authentication else { + unreachable!() + }; + let mut tampered_bytes = [0; ML_DSA_65_SIGNATURE_SIZE]; + tampered_bytes.copy_from_slice(signature.as_ref()); + tampered_bytes[0] ^= 1; + let tampered = + CarrierAuthenticationV1::MlDsa65(MlDsa65SignatureBytes::from_bytes(tampered_bytes)); + assert!(matches!( + context.verify_local_authentication_with_committee( + candidate.clone(), + tampered, + &committee_context, + CarrierAuthorizerV1::MlDsa65 { + authority: candidate.header().author(), + signer: &signer, + }, + ), + Err(RbcDagError::InvalidAuthentication) + )); + assert!(matches!( + context.verify_local_authentication_with_committee( + candidate.clone(), + authentication.clone(), + &committee_context, + CarrierAuthorizerV1::MlDsa65 { + authority: 2, + signer: &signer, + }, + ), + Err(RbcDagError::AuthorizerAuthorityMismatch { + expected: 3, + actual: 2, + }) + )); + + let wrong_context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xC9; 32]).unwrap(), + &committee_context, + BlockAuthenticationScheme::MlDsa65, + ); + assert!(matches!( + wrong_context.verify_local_authentication_with_committee( + candidate, + authentication, + &committee_context, + CarrierAuthorizerV1::MlDsa65 { + authority: 3, + signer: &signer, + }, + ), + Err(RbcDagError::InvalidAuthentication) + )); + } + + #[test] + fn persisted_local_mac_recovery_verifies_every_vector_entry_and_length() { + let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let candidate = + CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) + .unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xD9; 32]).unwrap(), + &committee_context, + BlockAuthenticationScheme::MacVector, + ); + let keyrings = mac_keyrings_for_test(committee.len()); + let author = candidate.header().author() as usize; + let authentication = context + .authenticate_with_committee( + &candidate, + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ) + .unwrap(); + context + .verify_local_authentication_with_committee( + candidate.clone(), + authentication.clone(), + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ) + .unwrap(); + + let CarrierAuthenticationV1::MacVector(vector) = &authentication else { + unreachable!() + }; + let mut poisoned_bytes = vector.as_bytes().to_vec(); + poisoned_bytes[2 * MAC_TAG_SIZE] ^= 1; + let poisoned = + CarrierAuthenticationV1::MacVector(FlatMacVector::from_bytes(poisoned_bytes).unwrap()); + context + .verify_authentication_with_committee( + candidate.clone(), + poisoned.clone(), + 1, + &committee_context, + &keyrings[1], + ) + .expect("a different recipient's entry remains valid"); + assert!(matches!( + context.verify_local_authentication_with_committee( + candidate.clone(), + poisoned, + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ), + Err(RbcDagError::InvalidAuthentication) + )); + + let short = CarrierAuthenticationV1::MacVector( + FlatMacVector::from_bytes( + vector.as_bytes()[..vector.as_bytes().len() - MAC_TAG_SIZE].to_vec(), + ) + .unwrap(), + ); + assert!(matches!( + context.verify_local_authentication_with_committee( + candidate, + short, + &committee_context, + CarrierAuthorizerV1::MacVector { + authority: author as AuthorityIndex, + keys: &keyrings[author], + }, + ), + Err(RbcDagError::InvalidMacVectorLength { .. }) + )); + } + #[test] fn sidecar_wire_has_frozen_flat_mac_shape() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index f1506bad..192385aa 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -19,6 +19,7 @@ use std::{ use crate::{ committee::Committee, + crypto::Blake3Hasher, types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, }; @@ -34,6 +35,9 @@ use super::{ pub const EXECUTABLE_MODEL_ADMISSION_WINDOW_V1: RoundNumber = 2; pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 4; +const MODEL_LINEAGE_DERIVE_CONTEXT: &str = "starfish-rbc-dag-model-lineage-v1"; +type ModelLineage = [u8; 32]; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum IngressAuthentication { Authenticated, @@ -59,6 +63,116 @@ pub enum ModelEffect { CarrierRoundAdvanced(RoundNumber), } +/// One proof-critical or externally observable step of a reducer transition. +/// +/// The order is part of the runtime contract. A caller may plan a transition +/// on a clone, persist these entries in order, and only then install the +/// planned model with [`RbcDagModel::commit_plan`]. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ModelTraceEvent { + /// The first authenticated value selected for a remote carrier slot. + AdmissionLocked(BlockReference), + /// A locally generated ECHO or READY became slot-global and immutable. + LocalPhaseLocked(RbcPhaseStatementV1), + /// One exact entry of an enclosing carrier's authenticated phase log is + /// about to be applied. Any lock enabled by that entry follows this event. + PhaseBatchEntryApplied { + outer: BlockReference, + index: usize, + sender: AuthorityIndex, + statement: RbcPhaseStatementV1, + }, + /// The entry at `index` was applied and the durable cursor may advance to + /// `next_index`. This always follows the matching application event. + PhaseBatchCursorAdvanced { + outer: BlockReference, + index: usize, + next_index: usize, + }, + /// The local author fixed one exact carrier before authorizing its ECHO. + LocalCarrierFixed(BlockReference), + /// Bracha delivery became slot-global and immutable. + DeliveryLocked(BlockReference), + /// Existing non-durable output retained in its exact reducer order. + Effect(ModelEffect), +} + +/// Ordered typed input from which the executable model can be reconstructed. +/// +/// `CandidateRetained` is ordinary candidate-only retention. The stricter +/// `CandidateRecovered` variant additionally requires prior phase evidence, +/// matching [`RbcDagModel::recover_carrier`]. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ModelInputRecord { + CandidateRetained(CandidateCarrierV1), + CandidateRecovered(CandidateCarrierV1), + AuthenticatedIngress(AuthenticatedCarrierV1), + LocalCarrierFixed(LocallyAuthenticatedCarrierV1), + DataAvailable(BlockReference), +} + +#[derive(Default)] +struct TransitionLog { + trace: Vec, +} + +impl TransitionLog { + fn proof(&mut self, event: ModelTraceEvent) { + self.trace.push(event); + } + + fn effect(&mut self, effect: ModelEffect) { + self.trace.push(ModelTraceEvent::Effect(effect)); + } + + fn effects(&self) -> Vec { + self.trace + .iter() + .filter_map(|entry| match entry { + ModelTraceEvent::Effect(effect) => Some(effect.clone()), + _ => None, + }) + .collect() + } +} + +/// A transition evaluated against an immutable model revision. +/// +/// Fields are deliberately private: the only way to install the planned state +/// is [`RbcDagModel::commit_plan`], which rejects a stale or foreign base. +#[derive(Clone)] +pub struct ModelTransitionPlan { + base_revision: u64, + base_lineage: ModelLineage, + base_context: RbcDagContextV1, + base_authority: AuthorityIndex, + input: ModelInputRecord, + trace: Vec, + next_model: RbcDagModel, +} + +impl ModelTransitionPlan { + /// The typed reducer input must be durably recorded before the ordered + /// proof trace is persisted and this plan is committed. + pub fn input(&self) -> &ModelInputRecord { + &self.input + } + + pub fn trace(&self) -> &[ModelTraceEvent] { + &self.trace + } + + pub fn effects(&self) -> Vec { + self.trace + .iter() + .filter_map(|entry| match entry { + ModelTraceEvent::Effect(effect) => Some(effect.clone()), + _ => None, + }) + .collect() + } +} + /// Snapshot of the lifecycle predicates for one exact carrier. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct CarrierLifecycle { @@ -122,6 +236,12 @@ pub enum ModelError { previous: Option, proposed: Option, }, + RevisionOverflow, + StaleTransitionPlan { + expected_revision: u64, + actual_revision: u64, + }, + ForeignTransitionPlan, } impl fmt::Display for ModelError { @@ -210,6 +330,8 @@ pub struct RbcDagModel { committee_id: RbcDagCommitteeId, context: RbcDagContextV1, own_authority: AuthorityIndex, + revision: u64, + lineage: ModelLineage, local_carrier_round: RoundNumber, own_fixed: BTreeMap, carriers: BTreeMap, @@ -248,6 +370,8 @@ impl RbcDagModel { committee_id, context, own_authority, + revision: 0, + lineage: [0; 32], local_carrier_round: 1, own_fixed: BTreeMap::new(), carriers: BTreeMap::new(), @@ -270,6 +394,127 @@ impl RbcDagModel { self.context } + /// Monotonic reducer revision used to reject a plan computed from stale + /// state. It is advanced once per successfully applied typed input record. + pub fn revision(&self) -> u64 { + self.revision + } + + /// Evaluate one typed input against a clone without changing live state. + /// A durable adapter records [`ModelTransitionPlan::input`] first, then + /// [`ModelTransitionPlan::trace`] in order, and commits only after both + /// writes are durable. + pub fn plan_input(&self, input: ModelInputRecord) -> Result { + let mut next_model = self.clone(); + let log = next_model.apply_input_traced(input.clone())?; + Ok(ModelTransitionPlan { + base_revision: self.revision, + base_lineage: self.lineage, + base_context: self.context, + base_authority: self.own_authority, + input, + trace: log.trace, + next_model, + }) + } + + /// Atomically install a previously planned state after its ordered trace + /// has been durably recorded by the caller. + pub fn commit_plan( + &mut self, + plan: ModelTransitionPlan, + ) -> Result, ModelError> { + if self.context != plan.base_context || self.own_authority != plan.base_authority { + return Err(ModelError::ForeignTransitionPlan); + } + if self.revision != plan.base_revision { + return Err(ModelError::StaleTransitionPlan { + expected_revision: plan.base_revision, + actual_revision: self.revision, + }); + } + if self.lineage != plan.base_lineage { + return Err(ModelError::ForeignTransitionPlan); + } + let effects = plan.effects(); + *self = plan.next_model; + Ok(effects) + } + + /// Apply one typed record immediately. Runtime adapters that need + /// write-ahead durability should use [`Self::plan_input`] and + /// [`Self::commit_plan`] instead. + pub fn apply_input(&mut self, input: ModelInputRecord) -> Result, ModelError> { + self.apply_input_traced(input).map(|log| log.effects()) + } + + /// Deterministically reconstruct a model by replaying the original typed + /// inputs in their recorded order. No round or lock is synthesized. + pub fn replay_from_records( + committee: Arc, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + records: I, + ) -> Result<(Self, Vec), ModelError> + where + I: IntoIterator, + { + let mut model = Self::new(committee, own_authority, context)?; + let mut trace = Vec::new(); + for record in records { + trace.extend(model.apply_input_traced(record)?.trace); + } + Ok((model, trace)) + } + + fn apply_input_traced(&mut self, input: ModelInputRecord) -> Result { + let next_revision = self + .revision + .checked_add(1) + .ok_or(ModelError::RevisionOverflow)?; + let next_lineage = self.input_lineage(&input); + let mut log = TransitionLog::default(); + match input { + ModelInputRecord::CandidateRetained(carrier) => { + self.receive_carrier_traced( + carrier, + IngressAuthentication::CandidateOnly, + &mut log, + )?; + } + ModelInputRecord::CandidateRecovered(carrier) => { + self.recover_carrier_traced(carrier, &mut log)?; + } + ModelInputRecord::AuthenticatedIngress(authenticated) => { + self.receive_authenticated_traced(authenticated, &mut log)?; + } + ModelInputRecord::LocalCarrierFixed(authenticated) => { + self.start_local_carrier_traced(authenticated, &mut log)?; + } + ModelInputRecord::DataAvailable(reference) => { + self.mark_data_available_traced(reference, &mut log)?; + } + } + self.lineage = next_lineage; + self.revision = next_revision; + Ok(log) + } + + fn input_lineage(&self, input: &ModelInputRecord) -> ModelLineage { + let (kind, reference) = match input { + ModelInputRecord::CandidateRetained(carrier) => (0, carrier.reference()), + ModelInputRecord::CandidateRecovered(carrier) => (1, carrier.reference()), + ModelInputRecord::AuthenticatedIngress(carrier) => (2, carrier.reference()), + ModelInputRecord::LocalCarrierFixed(carrier) => (3, carrier.reference()), + ModelInputRecord::DataAvailable(reference) => (4, *reference), + }; + let mut hasher = Blake3Hasher::new_derive_key(MODEL_LINEAGE_DERIVE_CONTEXT); + hasher.update(&self.lineage); + hasher.update(&[kind]); + update_lineage_reference(&mut hasher, reference); + hasher.finalize().into() + } + /// Current sequential local carrier slot. If `can_create_carrier` is /// false, the local carrier is fixed and waits for exact-round quorum. pub fn local_carrier_round(&self) -> RoundNumber { @@ -353,6 +598,14 @@ impl RbcDagModel { &mut self, authenticated: LocallyAuthenticatedCarrierV1, ) -> Result, ModelError> { + self.apply_input(ModelInputRecord::LocalCarrierFixed(authenticated)) + } + + fn start_local_carrier_traced( + &mut self, + authenticated: LocallyAuthenticatedCarrierV1, + log: &mut TransitionLog, + ) -> Result<(), ModelError> { self.ensure_locally_authenticated(&authenticated)?; let carrier = authenticated.candidate().clone(); self.ensure_committee(&carrier)?; @@ -408,6 +661,7 @@ impl RbcDagModel { // ECHO is authorized or any embedded phase statement is exposed. self.preflight_receive(&carrier)?; self.own_fixed.insert(round, reference); + log.proof(ModelTraceEvent::LocalCarrierFixed(reference)); for statement in &expected_phase_batch { self.pending_phase_set.remove(statement); } @@ -419,10 +673,9 @@ impl RbcDagModel { (!selected_phase_indices.contains(&index)).then_some(statement) }) .collect(); - let mut effects = - self.apply_received_carrier(carrier, IngressAuthentication::Authenticated); - self.maybe_advance_fast_clock(&mut effects); - Ok(effects) + self.apply_received_carrier(carrier, IngressAuthentication::Authenticated, log); + self.maybe_advance_fast_clock(log); + Ok(()) } /// Stage canonical content without granting optimistic admission or ECHO. @@ -430,7 +683,7 @@ impl RbcDagModel { &mut self, carrier: CandidateCarrierV1, ) -> Result, ModelError> { - self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) + self.apply_input(ModelInputRecord::CandidateRetained(carrier)) } /// Admit a carrier only through the opaque capability produced by the @@ -439,25 +692,36 @@ impl RbcDagModel { &mut self, authenticated: AuthenticatedCarrierV1, ) -> Result, ModelError> { + self.apply_input(ModelInputRecord::AuthenticatedIngress(authenticated)) + } + + fn receive_authenticated_traced( + &mut self, + authenticated: AuthenticatedCarrierV1, + log: &mut TransitionLog, + ) -> Result<(), ModelError> { self.ensure_authenticated(&authenticated)?; if authenticated.candidate().header().author() == self.own_authority { return Err(ModelError::LocalCarrierRequiresStart( authenticated.candidate().reference(), )); } - self.receive_carrier( + self.receive_carrier_traced( authenticated.candidate().clone(), IngressAuthentication::Authenticated, + log, ) } - fn receive_carrier( + fn receive_carrier_traced( &mut self, carrier: CandidateCarrierV1, authentication: IngressAuthentication, - ) -> Result, ModelError> { + log: &mut TransitionLog, + ) -> Result<(), ModelError> { self.preflight_receive(&carrier)?; - Ok(self.apply_received_carrier(carrier, authentication)) + self.apply_received_carrier(carrier, authentication, log); + Ok(()) } fn preflight_receive(&self, carrier: &CandidateCarrierV1) -> Result<(), ModelError> { @@ -490,16 +754,16 @@ impl RbcDagModel { &mut self, carrier: CandidateCarrierV1, authentication: IngressAuthentication, - ) -> Vec { + log: &mut TransitionLog, + ) { let reference = carrier.reference(); self.carriers .entry(reference) .or_insert_with(|| CarrierRecord::new(carrier)); - let mut effects = Vec::new(); // Canonical content can satisfy a previously latched recovery even if // the receiver-specific authenticator is invalid. - self.drive_rbc(reference, &mut effects); + self.drive_rbc(reference, log); if authentication == IngressAuthentication::Authenticated { self.carriers @@ -515,12 +779,11 @@ impl RbcDagModel { } }; if selected && self.in_admission_window(reference.round) { - self.promote_authenticated(reference, &mut effects); + self.promote_authenticated(reference, log); } } - self.maybe_advance_fast_clock(&mut effects); - self.drain_delivered_phase_batches(&mut effects); - effects + self.maybe_advance_fast_clock(log); + self.drain_delivered_phase_batches(log); } /// Accept an exact recovered carrier only after authenticated phase @@ -529,6 +792,14 @@ impl RbcDagModel { &mut self, carrier: CandidateCarrierV1, ) -> Result, ModelError> { + self.apply_input(ModelInputRecord::CandidateRecovered(carrier)) + } + + fn recover_carrier_traced( + &mut self, + carrier: CandidateCarrierV1, + log: &mut TransitionLog, + ) -> Result<(), ModelError> { self.ensure_committee(&carrier)?; let reference = carrier.reference(); let key = (reference.round, reference.authority); @@ -539,7 +810,7 @@ impl RbcDagModel { if !expected { return Err(ModelError::UnexpectedRecovery(reference)); } - self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) + self.receive_carrier_traced(carrier, IngressAuthentication::CandidateOnly, log) } /// Record transaction-data availability established by the external @@ -550,13 +821,20 @@ impl RbcDagModel { &mut self, reference: BlockReference, ) -> Result, ModelError> { + self.apply_input(ModelInputRecord::DataAvailable(reference)) + } + + fn mark_data_available_traced( + &mut self, + reference: BlockReference, + log: &mut TransitionLog, + ) -> Result<(), ModelError> { self.carriers .get_mut(&reference) .ok_or(ModelError::MissingCarrier(reference))? .data_available = true; - let mut effects = Vec::new(); - self.drive_prefix(reference.authority, &mut effects); - Ok(effects) + self.drive_prefix(reference.authority, log); + Ok(()) } pub fn lifecycle(&self, reference: &BlockReference) -> Option { @@ -593,6 +871,10 @@ impl RbcDagModel { &mut self, frontier: &[Option], ) -> Result, ModelError> { + let next_revision = self + .revision + .checked_add(1) + .ok_or(ModelError::RevisionOverflow)?; if frontier.len() != self.committee.len() { return Err(ModelError::FrontierLength { expected: self.committee.len(), @@ -622,9 +904,30 @@ impl RbcDagModel { } self.included_frontier.clone_from_slice(frontier); self.included.extend(delta.iter().copied()); + self.advance_frontier_lineage(frontier); + self.revision = next_revision; Ok(delta.into_iter().collect()) } + fn advance_frontier_lineage(&mut self, frontier: &[Option]) { + let mut hasher = Blake3Hasher::new_derive_key(MODEL_LINEAGE_DERIVE_CONTEXT); + hasher.update(&self.lineage); + hasher.update(&[5]); + hasher.update(&(frontier.len() as u64).to_be_bytes()); + for reference in frontier { + match reference { + Some(reference) => { + hasher.update(&[1]); + update_lineage_reference(&mut hasher, *reference); + } + None => { + hasher.update(&[0]); + } + } + } + self.lineage = hasher.finalize().into(); + } + fn collect_frontier_extension( &self, authority: AuthorityIndex, @@ -724,7 +1027,7 @@ impl RbcDagModel { .or_default() } - fn authorize_local_echo(&mut self, reference: BlockReference, effects: &mut Vec) { + fn authorize_local_echo(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; let slot = self.rbc_slot_mut(reference); if slot.echoed.is_some() { @@ -737,8 +1040,10 @@ impl RbcDagModel { .or_default() .echoes .insert(own); - self.queue_local_phase(RbcPhaseStatementV1::Echo { target: reference }); - self.drive_rbc(reference, effects); + let statement = RbcPhaseStatementV1::Echo { target: reference }; + log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); + self.queue_local_phase(statement); + self.drive_rbc(reference, log); } fn queue_local_phase(&mut self, statement: RbcPhaseStatementV1) { @@ -747,14 +1052,14 @@ impl RbcDagModel { } } - fn process_phase_batch(&mut self, outer: BlockReference, effects: &mut Vec) { - self.process_phase_batch_steps(outer, usize::MAX, effects); - self.drain_delivered_phase_batches(effects); + fn process_phase_batch(&mut self, outer: BlockReference, log: &mut TransitionLog) { + self.process_phase_batch_steps(outer, usize::MAX, log); + self.drain_delivered_phase_batches(log); } - fn drain_delivered_phase_batches(&mut self, effects: &mut Vec) { + fn drain_delivered_phase_batches(&mut self, log: &mut TransitionLog) { while let Some(outer) = self.pending_delivered_batch_replays.pop_front() { - self.process_phase_batch_steps(outer, usize::MAX, effects); + self.process_phase_batch_steps(outer, usize::MAX, log); } } @@ -762,32 +1067,48 @@ impl RbcDagModel { &mut self, outer: BlockReference, maximum_steps: usize, - effects: &mut Vec, + log: &mut TransitionLog, ) { let mut processed = 0; loop { if processed == maximum_steps { return; } - let Some((sender, statement)) = self.carriers.get(&outer).and_then(|record| { + let Some((index, sender, statement)) = self.carriers.get(&outer).and_then(|record| { + let index = record.phase_batch_cursor; record .carrier .header() .phase_batch() - .get(record.phase_batch_cursor) + .get(index) .copied() - .map(|statement| (record.carrier.header().author(), statement)) + .map(|statement| (index, record.carrier.header().author(), statement)) }) else { return; }; // Applying the statement is idempotent. Advance the persisted // cursor only afterwards, so a crash between the two replays the // same statement rather than skipping the unprocessed tail. - self.record_phase(sender, statement, effects); - self.carriers - .get_mut(&outer) - .expect("the outer carrier remains pinned") - .phase_batch_cursor += 1; + log.proof(ModelTraceEvent::PhaseBatchEntryApplied { + outer, + index, + sender, + statement, + }); + self.record_phase(sender, statement, log); + let next_index = { + let record = self + .carriers + .get_mut(&outer) + .expect("the outer carrier remains pinned"); + record.phase_batch_cursor += 1; + record.phase_batch_cursor + }; + log.proof(ModelTraceEvent::PhaseBatchCursorAdvanced { + outer, + index, + next_index, + }); processed += 1; } } @@ -796,7 +1117,7 @@ impl RbcDagModel { &mut self, sender: AuthorityIndex, statement: RbcPhaseStatementV1, - effects: &mut Vec, + log: &mut TransitionLog, ) { if !self.committee.known_authority(sender) { return; @@ -838,10 +1159,10 @@ impl RbcDagModel { candidate.readies.insert(sender); } } - self.drive_rbc(target, effects); + self.drive_rbc(target, log); } - fn drive_rbc(&mut self, target: BlockReference, effects: &mut Vec) { + fn drive_rbc(&mut self, target: BlockReference, log: &mut TransitionLog) { let slot_key = (target.round, target.authority); if !self .rbc_slots @@ -908,7 +1229,7 @@ impl RbcDagModel { .unwrap_or_default() .into_iter() .collect(); - effects.push(ModelEffect::NeedCarrier { target, holders }); + log.effect(ModelEffect::NeedCarrier { target, holders }); break; } RbcAction::SendReady => { @@ -921,7 +1242,9 @@ impl RbcDagModel { .or_default() .readies .insert(own); - self.queue_local_phase(RbcPhaseStatementV1::Ready { target }); + let statement = RbcPhaseStatementV1::Ready { target }; + log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); + self.queue_local_phase(statement); } RbcAction::Deliver => { self.rbc_slot_mut(target).delivered = Some(target); @@ -930,16 +1253,17 @@ impl RbcDagModel { .get_mut(&target) .expect("delivery requires exact canonical carrier content"); record.delivered = true; - effects.push(ModelEffect::Delivered(target)); + log.proof(ModelTraceEvent::DeliveryLocked(target)); + log.effect(ModelEffect::Delivered(target)); self.pending_delivered_batch_replays.push_back(target); - self.drive_prefix(target.authority, effects); + self.drive_prefix(target.authority, log); } RbcAction::None => break, } } } - fn maybe_advance_fast_clock(&mut self, effects: &mut Vec) { + fn maybe_advance_fast_clock(&mut self, log: &mut TransitionLog) { let round = self.local_carrier_round; if !self.own_fixed.contains_key(&round) { return; @@ -955,8 +1279,8 @@ impl RbcDagModel { return; } self.local_carrier_round = round.saturating_add(1); - effects.push(ModelEffect::CarrierRoundAdvanced(self.local_carrier_round)); - self.promote_buffered_window(effects); + log.effect(ModelEffect::CarrierRoundAdvanced(self.local_carrier_round)); + self.promote_buffered_window(log); } fn in_admission_window(&self, round: RoundNumber) -> bool { @@ -966,7 +1290,7 @@ impl RbcDagModel { .saturating_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) } - fn promote_authenticated(&mut self, reference: BlockReference, effects: &mut Vec) { + fn promote_authenticated(&mut self, reference: BlockReference, log: &mut TransitionLog) { let slot_key = (reference.round, reference.authority); if self.authenticated_by_slot.get(&slot_key) != Some(&reference) || self @@ -981,11 +1305,12 @@ impl RbcDagModel { .get_mut(&reference) .expect("an authenticated carrier remains staged") .admitted = true; - self.authorize_local_echo(reference, effects); - self.process_phase_batch(reference, effects); + log.proof(ModelTraceEvent::AdmissionLocked(reference)); + self.authorize_local_echo(reference, log); + self.process_phase_batch(reference, log); } - fn promote_buffered_window(&mut self, effects: &mut Vec) { + fn promote_buffered_window(&mut self, log: &mut TransitionLog) { let eligible: Vec<_> = self .authenticated_by_slot .values() @@ -993,11 +1318,11 @@ impl RbcDagModel { .filter(|reference| self.in_admission_window(reference.round)) .collect(); for reference in eligible { - self.promote_authenticated(reference, effects); + self.promote_authenticated(reference, log); } } - fn drive_prefix(&mut self, authority: AuthorityIndex, effects: &mut Vec) { + fn drive_prefix(&mut self, authority: AuthorityIndex, log: &mut TransitionLog) { loop { let Some(current_tip) = self.prefix_tips.get(authority as usize).copied() else { return; @@ -1021,7 +1346,7 @@ impl RbcDagModel { .expect("delivered carrier exists") .prefix_closed = true; self.prefix_tips[authority as usize] = next; - effects.push(ModelEffect::PrefixAdvanced { + log.effect(ModelEffect::PrefixAdvanced { authority, tip: next, }); @@ -1029,6 +1354,12 @@ impl RbcDagModel { } } +fn update_lineage_reference(hasher: &mut Blake3Hasher, reference: BlockReference) { + hasher.update(&reference.authority.to_be_bytes()); + hasher.update(&reference.round.to_be_bytes()); + hasher.update(reference.digest.as_array()); +} + #[cfg(test)] mod tests { use super::*; @@ -1232,6 +1563,345 @@ mod tests { all_honest_progress(7); } + #[test] + fn planned_authenticated_ingress_locks_admission_before_echo() { + let committee = committee(4); + let model = model(Arc::clone(&committee), 3); + let (own_prev, weak_parents) = genesis_parents(&committee, 0); + let carrier = + candidate(&committee, 0, 1, own_prev, weak_parents, Vec::new(), 0xD0).unwrap(); + let target = carrier.reference(); + let authenticated = authenticate_for(&committee, &carrier, 3); + let plan = model + .plan_input(ModelInputRecord::AuthenticatedIngress(authenticated)) + .unwrap(); + + let admission = plan + .trace() + .iter() + .position(|event| *event == ModelTraceEvent::AdmissionLocked(target)) + .unwrap(); + let echo = plan + .trace() + .iter() + .position(|event| { + *event == ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Echo { target }) + }) + .unwrap(); + assert!(admission < echo); + assert_eq!(model.revision(), 0); + assert!(model.lifecycle(&target).is_none()); + } + + #[test] + fn phase_application_precedes_ready_and_ready_precedes_delivery() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (own_prev, weak_parents) = genesis_parents(&committee, 0); + let target_carrier = + candidate(&committee, 0, 1, own_prev, weak_parents, Vec::new(), 0xD1).unwrap(); + let target = target_carrier.reference(); + model.stage_candidate(target_carrier).unwrap(); + // One remote READY is below V. The enclosing carrier supplies the + // second; the resulting local READY is then the third vote and delivers. + let mut setup = TransitionLog::default(); + model.record_phase(0, RbcPhaseStatementV1::Ready { target }, &mut setup); + + let outer = candidate( + &committee, + 1, + 2, + BlockReference::new_test(1, 1), + vec![ + BlockReference::new_test(0, 1), + BlockReference::new_test(2, 1), + ], + vec![RbcPhaseStatementV1::Ready { target }], + 0xD2, + ) + .unwrap(); + let outer_reference = outer.reference(); + let authenticated = authenticate_for(&committee, &outer, 3); + let plan = model + .plan_input(ModelInputRecord::AuthenticatedIngress(authenticated)) + .unwrap(); + let trace = plan.trace(); + + let applied = trace + .iter() + .position(|event| { + matches!( + event, + ModelTraceEvent::PhaseBatchEntryApplied { + outer, + index: 0, + sender: 1, + statement: RbcPhaseStatementV1::Ready { target: actual }, + } if *outer == outer_reference && *actual == target + ) + }) + .unwrap(); + let ready = trace + .iter() + .position(|event| { + *event == ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Ready { target }) + }) + .unwrap(); + let delivery = trace + .iter() + .position(|event| *event == ModelTraceEvent::DeliveryLocked(target)) + .unwrap(); + let cursor = trace + .iter() + .position(|event| { + matches!( + event, + ModelTraceEvent::PhaseBatchCursorAdvanced { + outer, + index: 0, + next_index: 1, + } if *outer == outer_reference + ) + }) + .unwrap(); + assert!(applied < ready); + assert!(ready < delivery); + assert!(delivery < cursor); + } + + #[test] + fn delivery_lock_precedes_replay_of_the_delivered_carrier_batch() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let replay_target = BlockReference::new_test(2, 1); + let target_carrier = candidate( + &committee, + 0, + 2, + BlockReference::new_test(0, 1), + vec![ + BlockReference::new_test(1, 1), + BlockReference::new_test(2, 1), + ], + vec![RbcPhaseStatementV1::Echo { + target: replay_target, + }], + 0xD3, + ) + .unwrap(); + let target = target_carrier.reference(); + model.stage_candidate(target_carrier).unwrap(); + let mut setup = TransitionLog::default(); + model.record_phase(0, RbcPhaseStatementV1::Ready { target }, &mut setup); + + let outer = candidate( + &committee, + 1, + 3, + BlockReference::new_test(1, 2), + vec![ + BlockReference::new_test(0, 2), + BlockReference::new_test(2, 2), + ], + vec![RbcPhaseStatementV1::Ready { target }], + 0xD4, + ) + .unwrap(); + let authenticated = authenticate_for(&committee, &outer, 3); + let plan = model + .plan_input(ModelInputRecord::AuthenticatedIngress(authenticated)) + .unwrap(); + let trace = plan.trace(); + let delivery = trace + .iter() + .position(|event| *event == ModelTraceEvent::DeliveryLocked(target)) + .unwrap(); + let replay = trace + .iter() + .position(|event| { + matches!( + event, + ModelTraceEvent::PhaseBatchEntryApplied { + outer, + index: 0, + sender: 0, + statement: RbcPhaseStatementV1::Echo { target: actual }, + } if *outer == target && *actual == replay_target + ) + }) + .unwrap(); + assert!(delivery < replay); + } + + #[test] + fn planned_transition_is_rollback_safe_and_rejects_stale_or_divergent_commits() { + let committee = committee(4); + let mut live = model(Arc::clone(&committee), 3); + let (own_prev, weak_parents) = genesis_parents(&committee, 0); + let carrier = + candidate(&committee, 0, 1, own_prev, weak_parents, Vec::new(), 0xD5).unwrap(); + let reference = carrier.reference(); + let authenticated = authenticate_for(&committee, &carrier, 3); + let plan = live + .plan_input(ModelInputRecord::AuthenticatedIngress( + authenticated.clone(), + )) + .unwrap(); + + // Planning and dropping a clone cannot expose any live transition. + assert_eq!(live.revision(), 0); + assert!(live.lifecycle(&reference).is_none()); + let mut committed = live.clone(); + committed.commit_plan(plan.clone()).unwrap(); + assert_eq!(committed.revision(), 1); + assert!(committed.lifecycle(&reference).unwrap().admitted); + assert!(live.lifecycle(&reference).is_none()); + + // Any intervening successful input invalidates the old base revision. + live.stage_candidate(carrier).unwrap(); + assert_eq!(live.revision(), 1); + assert_eq!( + live.commit_plan(plan), + Err(ModelError::StaleTransitionPlan { + expected_revision: 0, + actual_revision: 1, + }) + ); + let lifecycle = live.lifecycle(&reference).unwrap(); + assert!(!lifecycle.authenticated); + assert!(!lifecycle.admitted); + + // Equal revision numbers do not make independently evolved clones + // interchangeable: their private lineages bind a plan to its exact + // base state. + let divergent_plan = committed + .plan_input(ModelInputRecord::DataAvailable(reference)) + .unwrap(); + assert_eq!( + live.commit_plan(divergent_plan), + Err(ModelError::ForeignTransitionPlan) + ); + let lifecycle = live.lifecycle(&reference).unwrap(); + assert!(!lifecycle.authenticated); + assert!(!lifecycle.data_available); + } + + #[test] + fn ordered_typed_replay_reconstructs_rounds_and_local_locks() { + let committee = committee(4); + let context = context(&committee); + let mut live = RbcDagModel::new(Arc::clone(&committee), 3, context).unwrap(); + let mut records = Vec::new(); + let mut references = Vec::new(); + + let (own_prev, weak_parents) = live.local_parent_set().unwrap(); + let local_one = candidate( + &committee, + 3, + 1, + own_prev, + weak_parents, + live.pending_phase_batch(), + 0xE0, + ) + .unwrap(); + let local_one_record = + ModelInputRecord::LocalCarrierFixed(authenticate_local(&committee, &local_one)); + live.apply_input(local_one_record.clone()).unwrap(); + records.push(local_one_record); + references.push(local_one.reference()); + + let mut round_one = BTreeMap::new(); + for (author, marker) in [(0, 0xE1), (1, 0xE2)] { + let (own_prev, weak_parents) = genesis_parents(&committee, author); + let carrier = candidate( + &committee, + author, + 1, + own_prev, + weak_parents, + Vec::new(), + marker, + ) + .unwrap(); + if author == 0 { + let retained = ModelInputRecord::CandidateRetained(carrier.clone()); + live.apply_input(retained.clone()).unwrap(); + records.push(retained); + } + let ingress = + ModelInputRecord::AuthenticatedIngress(authenticate_for(&committee, &carrier, 3)); + live.apply_input(ingress.clone()).unwrap(); + records.push(ingress); + references.push(carrier.reference()); + round_one.insert(author, carrier); + } + assert_eq!(live.local_carrier_round(), 2); + + let (own_prev, weak_parents) = live.local_parent_set().unwrap(); + let local_two = candidate( + &committee, + 3, + 2, + own_prev, + weak_parents, + live.pending_phase_batch(), + 0xE3, + ) + .unwrap(); + let local_two_record = + ModelInputRecord::LocalCarrierFixed(authenticate_local(&committee, &local_two)); + live.apply_input(local_two_record.clone()).unwrap(); + records.push(local_two_record); + references.push(local_two.reference()); + + for (author, marker, other) in [(0, 0xE4, 1), (1, 0xE5, 0)] { + let carrier = candidate( + &committee, + author, + 2, + round_one[&author].reference(), + vec![round_one[&other].reference(), local_one.reference()], + Vec::new(), + marker, + ) + .unwrap(); + let ingress = + ModelInputRecord::AuthenticatedIngress(authenticate_for(&committee, &carrier, 3)); + live.apply_input(ingress.clone()).unwrap(); + records.push(ingress); + references.push(carrier.reference()); + } + assert_eq!(live.local_carrier_round(), 3); + + let (replayed, trace) = + RbcDagModel::replay_from_records(Arc::clone(&committee), 3, context, records.clone()) + .unwrap(); + assert!(!trace.is_empty()); + assert_eq!(replayed.revision(), records.len() as u64); + assert_eq!(replayed.lineage, live.lineage); + assert_eq!(replayed.local_carrier_round, live.local_carrier_round); + assert_eq!(replayed.own_fixed, live.own_fixed); + assert_eq!(replayed.authenticated_by_slot, live.authenticated_by_slot); + assert_eq!(replayed.admitted_by_slot, live.admitted_by_slot); + assert_eq!(replayed.rbc_slots, live.rbc_slots); + assert_eq!(replayed.pending_phases, live.pending_phases); + assert_eq!(replayed.pending_phase_set, live.pending_phase_set); + for reference in references { + assert_eq!(replayed.lifecycle(&reference), live.lifecycle(&reference)); + } + + // Recovery is sequential: retaining only the round-two local record + // cannot synthesize round one or jump the local carrier clock. + assert!(matches!( + RbcDagModel::replay_from_records(committee, 3, context, [records[4].clone()],), + Err(ModelError::UnexpectedLocalRound { + expected: 1, + actual: 2, + }) + )); + } + #[test] fn phase_backlog_exposes_only_a_bounded_fifo_prefix() { let committee = committee(4); @@ -1302,10 +1972,10 @@ mod tests { sender: AuthorityIndex, statement: RbcPhaseStatementV1, ) -> Vec { - let mut effects = Vec::new(); - model.record_phase(sender, statement, &mut effects); - model.drain_delivered_phase_batches(&mut effects); - effects + let mut log = TransitionLog::default(); + model.record_phase(sender, statement, &mut log); + model.drain_delivered_phase_batches(&mut log); + log.effects() } fn force_deliver(model: &mut RbcDagModel, carrier: CandidateCarrierV1) { @@ -1785,7 +2455,7 @@ mod tests { model.stage_candidate(outer).unwrap(); let mut uninterrupted = model.clone(); - uninterrupted.process_phase_batch(outer_ref, &mut Vec::new()); + uninterrupted.process_phase_batch(outer_ref, &mut TransitionLog::default()); // Model a crash after the first idempotent statement was persisted but // before the outer batch cursor was advanced. @@ -1793,9 +2463,9 @@ mod tests { restarted.record_phase( 0, RbcPhaseStatementV1::Echo { target: first }, - &mut Vec::new(), + &mut TransitionLog::default(), ); - restarted.process_phase_batch(outer_ref, &mut Vec::new()); + restarted.process_phase_batch(outer_ref, &mut TransitionLog::default()); assert_eq!(restarted.rbc_slots, uninterrupted.rbc_slots); assert_eq!(restarted.pending_phases, uninterrupted.pending_phases); @@ -1855,8 +2525,7 @@ mod tests { model .pending_delivered_batch_replays .push_back(*references.last().unwrap()); - let mut effects = Vec::new(); - model.drain_delivered_phase_batches(&mut effects); + model.drain_delivered_phase_batches(&mut TransitionLog::default()); assert!(model.pending_delivered_batch_replays.is_empty()); assert_eq!(model.delivered(0, 1), Some(references[0])); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 86fc5274..1bda66fe 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -22,7 +22,7 @@ use crate::{ use super::{ CandidateCarrierV1, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, - RbcDagCommitteeId, RbcDagProjectionError, carrier_genesis_reference, + RbcDagCommitteeContextV1, RbcDagProjectionError, carrier_genesis_reference, }; /// An indexed exact carrier-prefix frontier. `None` is the authority's virtual @@ -135,7 +135,7 @@ struct ProjectedVertex { #[derive(Clone)] pub struct CertifiedProjectionModel { committee: Arc, - committee_id: RbcDagCommitteeId, + committee_context: RbcDagCommitteeContextV1, carriers: BTreeMap, delivered_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, closed_prefixes: Vec>, @@ -146,13 +146,21 @@ pub struct CertifiedProjectionModel { } impl CertifiedProjectionModel { + /// Convenience constructor that validates and hashes the committee once. + /// Runtime code that already owns the reusable capability should call + /// [`Self::from_committee_context`]. pub fn new(committee: Arc) -> Result { - let committee_id = RbcDagCommitteeId::derive(&committee) + let committee = RbcDagCommitteeContextV1::new(committee) .map_err(|_| CertifiedProjectionError::CommitteeMismatch)?; - let committee_size = committee.len(); - Ok(Self { - committee, - committee_id, + Ok(Self::from_committee_context(committee)) + } + + pub fn from_committee_context(committee: RbcDagCommitteeContextV1) -> Self { + let committee_size = committee.committee().len(); + let committee_arc = committee.committee_arc(); + Self { + committee: committee_arc, + committee_context: committee, carriers: BTreeMap::new(), delivered_slots: BTreeMap::new(), closed_prefixes: vec![Vec::new(); committee_size], @@ -160,7 +168,7 @@ impl CertifiedProjectionModel { consensus_slots: BTreeMap::new(), committed_frontier: vec![None; committee_size], committed_anchors: BTreeSet::new(), - }) + } } /// Retain a canonical carrier independently of optional-vertex validity. @@ -168,7 +176,7 @@ impl CertifiedProjectionModel { &mut self, candidate: CandidateCarrierV1, ) -> Result<(), CertifiedProjectionError> { - if candidate.committee_id() != self.committee_id { + if candidate.committee_id() != self.committee_context.committee_id() { return Err(CertifiedProjectionError::CommitteeMismatch); } self.carriers @@ -293,7 +301,7 @@ impl CertifiedProjectionModel { state .candidate - .validate_consensus_vertex(&self.committee) + .validate_consensus_vertex_with_committee(&self.committee_context) .map_err(CertifiedProjectionError::InvalidProjectionShape)?; if !state.delivered { return Err(CertifiedProjectionError::CarrierNotDelivered( diff --git a/crates/starfish-core/src/starfish_rbc_dag/storage.rs b/crates/starfish-core/src/starfish_rbc_dag/storage.rs new file mode 100644 index 00000000..d553658a --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag/storage.rs @@ -0,0 +1,1388 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Durable, opaque-record WAL for Starfish-RBC-DAG shadow state. +//! +//! The storage layer deliberately does not encode [`super::journal::JournalEventV1`]. +//! A later integration layer owns that versioned codec and the recovery of +//! opaque authentication capabilities. This module supplies the durability +//! boundary underneath it: one batch is one checksummed frame, and a caller +//! may expose the corresponding effects only after [`ShadowWalV1::append_batch`] +//! returns successfully. +//! +//! Recovery discards only a physically short final frame. A fully present +//! frame with a bad header, commit marker, or checksum is reported as +//! corruption even at end-of-file, so acknowledged proof-critical state is +//! never silently erased. + +use std::{ + error::Error, + ffi::OsString, + fmt, + fs::{self, File, OpenOptions}, + io::{self, Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +#[cfg(unix)] +use std::{ffi::CString, os::unix::ffi::OsStrExt}; + +use crate::types::AuthorityIndex; + +use super::RbcDagContextV1; + +const FILE_MAGIC: &[u8; 16] = b"STRFSH_RBCDAGWAL"; +const FRAME_MAGIC: u32 = 0x5242_4446; // "RBDF" +const FRAME_COMMIT_MAGIC: u32 = 0x434F_4D54; // "COMT" + +pub const SHADOW_WAL_FORMAT_VERSION_V1: u16 = 1; +pub const MAX_SHADOW_WAL_RECORD_SIZE_V1: usize = 16 * 1024 * 1024; +pub const MAX_SHADOW_WAL_BATCH_RECORDS_V1: usize = 4_096; +pub const MAX_SHADOW_WAL_FRAME_PAYLOAD_V1: usize = 64 * 1024 * 1024; + +const FILE_HEADER_PREFIX_LEN: usize = 16 + 2 + 2 + 32 + 32 + 2 + 2; +const FILE_HEADER_LEN: usize = FILE_HEADER_PREFIX_LEN + 4; +const FRAME_HEADER_PREFIX_LEN: usize = 4 + 2 + 2 + 8 + 8 + 4 + 4; +const FRAME_HEADER_LEN: usize = FRAME_HEADER_PREFIX_LEN + 4; +const FRAME_TRAILER_LEN: usize = 4 + 4; +const MAX_INITIALIZATION_TEMP_ATTEMPTS: usize = 128; + +static INITIALIZATION_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ShadowWalNamespaceV1 { + protocol_instance: [u8; 32], + committee_id: [u8; 32], + own_authority: AuthorityIndex, +} + +impl ShadowWalNamespaceV1 { + pub fn new(context: RbcDagContextV1, own_authority: AuthorityIndex) -> Self { + Self { + protocol_instance: *context.protocol_instance().as_bytes(), + committee_id: *context.committee_id().as_bytes(), + own_authority, + } + } + + pub fn protocol_instance(&self) -> &[u8; 32] { + &self.protocol_instance + } + + pub fn committee_id(&self) -> &[u8; 32] { + &self.committee_id + } + + pub fn own_authority(&self) -> AuthorityIndex { + self.own_authority + } + + pub fn format_version(&self) -> u16 { + SHADOW_WAL_FORMAT_VERSION_V1 + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RecoveredBatchV1 { + sequence: u64, + start_offset: u64, + end_offset: u64, + records: Vec>, +} + +impl RecoveredBatchV1 { + pub fn sequence(&self) -> u64 { + self.sequence + } + + pub fn start_offset(&self) -> u64 { + self.start_offset + } + + pub fn end_offset(&self) -> u64 { + self.end_offset + } + + pub fn records(&self) -> &[Vec] { + &self.records + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ShadowWalRecoveryV1 { + batches: Vec, + durable_file_len: u64, + record_count: u64, + discarded_tail_bytes: u64, +} + +impl ShadowWalRecoveryV1 { + pub fn batches(&self) -> &[RecoveredBatchV1] { + &self.batches + } + + pub fn batch_count(&self) -> u64 { + u64::try_from(self.batches.len()).expect("batch count always fits u64") + } + + pub fn record_count(&self) -> u64 { + self.record_count + } + + pub fn durable_file_len(&self) -> u64 { + self.durable_file_len + } + + pub fn discarded_tail_bytes(&self) -> u64 { + self.discarded_tail_bytes + } + + pub fn records(&self) -> Vec<&[u8]> { + self.batches + .iter() + .flat_map(|batch| batch.records.iter().map(Vec::as_slice)) + .collect() + } + + pub fn into_records(self) -> Vec> { + self.batches + .into_iter() + .flat_map(|batch| batch.records) + .collect() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DurableBatchPositionV1 { + sequence: u64, + start_offset: u64, + end_offset: u64, + record_count: u32, +} + +impl DurableBatchPositionV1 { + pub fn sequence(&self) -> u64 { + self.sequence + } + + pub fn start_offset(&self) -> u64 { + self.start_offset + } + + pub fn end_offset(&self) -> u64 { + self.end_offset + } + + pub fn record_count(&self) -> u32 { + self.record_count + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ShadowWalSummaryV1 { + file_len: u64, + batch_count: u64, + record_count: u64, +} + +impl ShadowWalSummaryV1 { + pub fn file_len(&self) -> u64 { + self.file_len + } + + pub fn batch_count(&self) -> u64 { + self.batch_count + } + + pub fn record_count(&self) -> u64 { + self.record_count + } +} + +#[derive(Debug)] +pub enum ShadowWalErrorV1 { + Io(io::Error), + TruncatedFileHeader { + actual: u64, + expected: usize, + }, + InvalidFileMagic, + UnsupportedFileVersion(u16), + InvalidFileHeaderLength(u16), + InvalidFileHeaderFlags(u16), + InvalidFileHeaderChecksum, + NamespaceMismatch, + EmptyBatch, + TooManyRecords(usize), + RecordTooLarge(usize), + FramePayloadTooLarge(usize), + LengthOverflow, + CorruptFrame { + offset: u64, + reason: &'static str, + }, + UnexpectedFrameSequence { + offset: u64, + expected: u64, + actual: u64, + }, + ExternalFileMutation { + expected: u64, + actual: u64, + }, + Poisoned, +} + +impl fmt::Display for ShadowWalErrorV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(error) => write!(formatter, "Starfish-RBC-DAG shadow WAL I/O error: {error}"), + other => write!(formatter, "Starfish-RBC-DAG shadow WAL error: {other:?}"), + } + } +} + +impl Error for ShadowWalErrorV1 { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(error) => Some(error), + _ => None, + } + } +} + +impl From for ShadowWalErrorV1 { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +pub struct ShadowWalV1 { + path: PathBuf, + file: File, + namespace: ShadowWalNamespaceV1, + durable_file_len: u64, + batch_count: u64, + record_count: u64, + poisoned: bool, +} + +impl ShadowWalV1 { + /// Open the WAL for exclusive single-writer use and replay every complete + /// durable batch in sequence order. + /// + /// The caller must ensure that no other writer mutates `path` while this + /// handle is alive. A changed file length is detected before an append, + /// but this adapter intentionally does not provide cross-process locking. + pub fn open( + path: impl AsRef, + namespace: ShadowWalNamespaceV1, + ) -> Result<(Self, ShadowWalRecoveryV1), ShadowWalErrorV1> { + let path = path.as_ref().to_path_buf(); + if let Some(parent) = nonempty_parent(&path) { + create_parent_directories_durable(parent)?; + } + + let mut file = loop { + match OpenOptions::new().read(true).write(true).open(&path) { + Ok(file) => break file, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + match fs::symlink_metadata(&path) { + Ok(_) => { + // A directory entry (for example a dangling + // symlink) already exists. Treat it as malformed; + // otherwise a no-replace publication retry would + // spin forever and might later mask the entry. + return Err(error.into()); + } + Err(metadata_error) if metadata_error.kind() == io::ErrorKind::NotFound => { + } + Err(metadata_error) => return Err(metadata_error.into()), + } + if let Some(file) = initialize_new_wal(&path, namespace)? { + break file; + } + // Another initializer atomically published `path` first. + // Open and validate exactly what won the race; never + // replace or repair it here. + } + Err(error) => return Err(error.into()), + } + }; + + let recovery = recover_file(&mut file, namespace)?; + file.seek(SeekFrom::Start(recovery.durable_file_len))?; + let wal = Self { + path, + file, + namespace, + durable_file_len: recovery.durable_file_len, + batch_count: recovery.batch_count(), + record_count: recovery.record_count, + poisoned: false, + }; + Ok((wal, recovery)) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn namespace(&self) -> ShadowWalNamespaceV1 { + self.namespace + } + + pub fn file_len(&self) -> u64 { + self.durable_file_len + } + + pub fn batch_count(&self) -> u64 { + self.batch_count + } + + pub fn record_count(&self) -> u64 { + self.record_count + } + + pub fn is_poisoned(&self) -> bool { + self.poisoned + } + + /// Append and fsync one atomic record batch before returning. + /// + /// Any seek, write, or fsync failure poisons this handle because the commit + /// result may be ambiguous. Drop it and reopen the WAL; recovery will + /// either retain the complete frame or discard its physically short torn + /// suffix. + pub fn append_batch( + &mut self, + records: &[Vec], + ) -> Result { + if self.poisoned { + return Err(ShadowWalErrorV1::Poisoned); + } + let sequence = self.batch_count; + let frame = encode_frame(sequence, records)?; + let added_records = + u64::try_from(records.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; + let next_batch_count = self + .batch_count + .checked_add(1) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + let next_record_count = self + .record_count + .checked_add(added_records) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + let durable_record_count = + u32::try_from(records.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; + let actual_len = self.file.metadata()?.len(); + if actual_len != self.durable_file_len { + self.poisoned = true; + return Err(ShadowWalErrorV1::ExternalFileMutation { + expected: self.durable_file_len, + actual: actual_len, + }); + } + let start_offset = self.durable_file_len; + let frame_len = u64::try_from(frame.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; + let end_offset = start_offset + .checked_add(frame_len) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + + if let Err(error) = self + .file + .seek(SeekFrom::Start(start_offset)) + .and_then(|_| self.file.write_all(&frame)) + .and_then(|_| self.file.sync_all()) + { + self.poisoned = true; + return Err(ShadowWalErrorV1::Io(error)); + } + + self.durable_file_len = end_offset; + self.batch_count = next_batch_count; + self.record_count = next_record_count; + Ok(DurableBatchPositionV1 { + sequence, + start_offset, + end_offset, + record_count: durable_record_count, + }) + } + + pub fn summary(&self) -> ShadowWalSummaryV1 { + ShadowWalSummaryV1 { + file_len: self.durable_file_len, + batch_count: self.batch_count, + record_count: self.record_count, + } + } + + /// Flush file metadata and close this writer by consuming it. + pub fn shutdown(self) -> Result { + self.file.sync_all()?; + if self.poisoned { + return Err(ShadowWalErrorV1::Poisoned); + } + Ok(self.summary()) + } +} + +/// Build a complete, durable header away from the canonical path, then +/// publish it without replacing any concurrently-created target. +/// +/// `Ok(None)` means another initializer won the publication race. The caller +/// must open and validate that target rather than assuming it is compatible. +fn initialize_new_wal( + path: &Path, + namespace: ShadowWalNamespaceV1, +) -> Result, ShadowWalErrorV1> { + let (temporary_path, mut file) = create_initialization_temp(path)?; + let cleanup = InitializationTempCleanup::new(temporary_path.clone()); + file.write_all(&encode_file_header(namespace))?; + file.sync_all()?; + + match atomic_rename_noreplace(&temporary_path, path) { + Ok(()) => { + // The rename makes the complete inode visible atomically; the + // directory sync makes that name durable across a power loss. + sync_parent_directory(path)?; + cleanup.disarm(); + Ok(Some(file)) + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => Ok(None), + Err(error) => Err(error.into()), + } +} + +fn create_initialization_temp(path: &Path) -> io::Result<(PathBuf, File)> { + let parent = nonempty_parent(path).unwrap_or_else(|| Path::new(".")); + let file_name = path.file_name().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "shadow WAL path has no file name", + ) + })?; + for _ in 0..MAX_INITIALIZATION_TEMP_ATTEMPTS { + let counter = INITIALIZATION_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + let mut temporary_name = OsString::from("."); + temporary_name.push(file_name); + temporary_name.push(format!(".init-{}-{counter}.tmp", std::process::id())); + let temporary_path = parent.join(temporary_name); + match OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&temporary_path) + { + Ok(file) => return Ok((temporary_path, file)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "could not allocate a unique shadow WAL initialization file", + )) +} + +struct InitializationTempCleanup { + path: PathBuf, + armed: bool, +} + +impl InitializationTempCleanup { + fn new(path: PathBuf) -> Self { + Self { path, armed: true } + } + + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for InitializationTempCleanup { + fn drop(&mut self) { + if self.armed { + let _ = fs::remove_file(&self.path); + } + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { + let from = path_to_c_string(from)?; + let to = path_to_c_string(to)?; + // SAFETY: both paths are live NUL-terminated C strings for the duration + // of the call. RENAME_NOREPLACE gives the required atomic no-clobber + // publication semantics. + let result = unsafe { + libc::renameat2( + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + if result == 0 { + Ok(()) + } else { + let error = io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(code) + if code == libc::ENOSYS + || code == libc::EINVAL + || code == libc::ENOTSUP + || code == libc::EOPNOTSUPP) + { + atomic_link_noreplace(from.as_bytes(), to.as_bytes()) + } else { + Err(error) + } + } +} + +#[cfg(target_vendor = "apple")] +fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { + let from = path_to_c_string(from)?; + let to = path_to_c_string(to)?; + // SAFETY: both paths are live NUL-terminated C strings for the duration + // of the call. RENAME_EXCL prevents replacement of an existing target. + let result = unsafe { libc::renamex_np(from.as_ptr(), to.as_ptr(), libc::RENAME_EXCL) }; + if result == 0 { + Ok(()) + } else { + let error = io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(code) if code == libc::ENOTSUP || code == libc::EINVAL) + { + atomic_link_noreplace(from.as_bytes(), to.as_bytes()) + } else { + Err(error) + } + } +} + +#[cfg(unix)] +fn path_to_c_string(path: &Path) -> io::Result { + CString::new(path.as_os_str().as_bytes()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "shadow WAL path contains an interior NUL", + ) + }) +} + +#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))] +fn atomic_link_noreplace(from: &[u8], to: &[u8]) -> io::Result<()> { + let from = Path::new(std::ffi::OsStr::from_bytes(from)); + let to = Path::new(std::ffi::OsStr::from_bytes(to)); + fs::hard_link(from, to)?; + // The target now atomically names the fully-synced inode. Failure to + // remove the private temporary name is harmless to WAL correctness. + let _ = fs::remove_file(from); + Ok(()) +} + +#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))] +fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { + // Portable fallback for platforms without a no-replace rename primitive: + // same-directory hard-link creation is still an atomic no-clobber publish. + fs::hard_link(from, to)?; + let _ = fs::remove_file(from); + Ok(()) +} + +fn encode_file_header(namespace: ShadowWalNamespaceV1) -> [u8; FILE_HEADER_LEN] { + let mut header = [0u8; FILE_HEADER_LEN]; + header[0..16].copy_from_slice(FILE_MAGIC); + header[16..18].copy_from_slice(&SHADOW_WAL_FORMAT_VERSION_V1.to_be_bytes()); + header[18..20].copy_from_slice(&(FILE_HEADER_LEN as u16).to_be_bytes()); + header[20..52].copy_from_slice(&namespace.protocol_instance); + header[52..84].copy_from_slice(&namespace.committee_id); + header[84..86].copy_from_slice(&namespace.own_authority.to_be_bytes()); + header[86..88].copy_from_slice(&0u16.to_be_bytes()); + let checksum = crc32c(&header[..FILE_HEADER_PREFIX_LEN]); + header[FILE_HEADER_PREFIX_LEN..FILE_HEADER_LEN].copy_from_slice(&checksum.to_be_bytes()); + header +} + +fn validate_file_header( + header: &[u8; FILE_HEADER_LEN], + expected_namespace: ShadowWalNamespaceV1, +) -> Result<(), ShadowWalErrorV1> { + if &header[0..16] != FILE_MAGIC { + return Err(ShadowWalErrorV1::InvalidFileMagic); + } + let version = read_u16(&header[16..18]); + if version != SHADOW_WAL_FORMAT_VERSION_V1 { + return Err(ShadowWalErrorV1::UnsupportedFileVersion(version)); + } + let header_len = read_u16(&header[18..20]); + if usize::from(header_len) != FILE_HEADER_LEN { + return Err(ShadowWalErrorV1::InvalidFileHeaderLength(header_len)); + } + let flags = read_u16(&header[86..88]); + if flags != 0 { + return Err(ShadowWalErrorV1::InvalidFileHeaderFlags(flags)); + } + let checksum = read_u32(&header[FILE_HEADER_PREFIX_LEN..FILE_HEADER_LEN]); + if checksum != crc32c(&header[..FILE_HEADER_PREFIX_LEN]) { + return Err(ShadowWalErrorV1::InvalidFileHeaderChecksum); + } + let actual_namespace = ShadowWalNamespaceV1 { + protocol_instance: header[20..52] + .try_into() + .expect("fixed protocol-instance range"), + committee_id: header[52..84].try_into().expect("fixed committee range"), + own_authority: read_u16(&header[84..86]), + }; + if actual_namespace != expected_namespace { + return Err(ShadowWalErrorV1::NamespaceMismatch); + } + Ok(()) +} + +fn recover_file( + file: &mut File, + namespace: ShadowWalNamespaceV1, +) -> Result { + let original_len = file.metadata()?.len(); + if original_len < FILE_HEADER_LEN as u64 { + return Err(ShadowWalErrorV1::TruncatedFileHeader { + actual: original_len, + expected: FILE_HEADER_LEN, + }); + } + file.seek(SeekFrom::Start(0))?; + let mut file_header = [0u8; FILE_HEADER_LEN]; + file.read_exact(&mut file_header)?; + validate_file_header(&file_header, namespace)?; + + let mut batches = Vec::new(); + let mut record_count = 0u64; + let mut offset = FILE_HEADER_LEN as u64; + let mut expected_sequence = 0u64; + while offset < original_len { + let remaining = original_len - offset; + if remaining < FRAME_HEADER_LEN as u64 { + break; + } + + file.seek(SeekFrom::Start(offset))?; + let mut frame_header = [0u8; FRAME_HEADER_LEN]; + file.read_exact(&mut frame_header)?; + validate_frame_header(&frame_header, offset)?; + let sequence = read_u64(&frame_header[8..16]); + if sequence != expected_sequence { + return Err(ShadowWalErrorV1::UnexpectedFrameSequence { + offset, + expected: expected_sequence, + actual: sequence, + }); + } + let payload_len_u64 = read_u64(&frame_header[16..24]); + let payload_len = + usize::try_from(payload_len_u64).map_err(|_| ShadowWalErrorV1::CorruptFrame { + offset, + reason: "payload length does not fit usize", + })?; + if payload_len > MAX_SHADOW_WAL_FRAME_PAYLOAD_V1 { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "payload exceeds configured maximum", + }); + } + let frame_len = (FRAME_HEADER_LEN as u64) + .checked_add(payload_len_u64) + .and_then(|length| length.checked_add(FRAME_TRAILER_LEN as u64)) + .ok_or(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "frame length overflow", + })?; + if frame_len > remaining { + break; + } + + let mut payload = vec![0u8; payload_len]; + file.read_exact(&mut payload)?; + let mut trailer = [0u8; FRAME_TRAILER_LEN]; + file.read_exact(&mut trailer)?; + if read_u32(&trailer[4..8]) != FRAME_COMMIT_MAGIC { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "invalid frame commit marker", + }); + } + let mut checksum_bytes = Vec::with_capacity(FRAME_HEADER_LEN + payload.len()); + checksum_bytes.extend_from_slice(&frame_header); + checksum_bytes.extend_from_slice(&payload); + if read_u32(&trailer[0..4]) != crc32c(&checksum_bytes) { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "frame checksum mismatch", + }); + } + + let declared_records = read_u32(&frame_header[24..28]); + let records = decode_records(&payload, declared_records, offset)?; + record_count = record_count + .checked_add(u64::from(declared_records)) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + let end_offset = offset + frame_len; + batches.push(RecoveredBatchV1 { + sequence, + start_offset: offset, + end_offset, + records, + }); + offset = end_offset; + expected_sequence = expected_sequence + .checked_add(1) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + } + + let discarded_tail_bytes = original_len - offset; + if discarded_tail_bytes != 0 { + file.set_len(offset)?; + file.sync_all()?; + } + Ok(ShadowWalRecoveryV1 { + batches, + durable_file_len: offset, + record_count, + discarded_tail_bytes, + }) +} + +fn encode_frame(sequence: u64, records: &[Vec]) -> Result, ShadowWalErrorV1> { + if records.is_empty() { + return Err(ShadowWalErrorV1::EmptyBatch); + } + if records.len() > MAX_SHADOW_WAL_BATCH_RECORDS_V1 { + return Err(ShadowWalErrorV1::TooManyRecords(records.len())); + } + let mut payload_len = 0usize; + for record in records { + if record.len() > MAX_SHADOW_WAL_RECORD_SIZE_V1 { + return Err(ShadowWalErrorV1::RecordTooLarge(record.len())); + } + payload_len = payload_len + .checked_add(4) + .and_then(|length| length.checked_add(record.len())) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + } + if payload_len > MAX_SHADOW_WAL_FRAME_PAYLOAD_V1 { + return Err(ShadowWalErrorV1::FramePayloadTooLarge(payload_len)); + } + let payload_len_u64 = + u64::try_from(payload_len).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; + let record_count = + u32::try_from(records.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; + + let total_len = FRAME_HEADER_LEN + .checked_add(payload_len) + .and_then(|length| length.checked_add(FRAME_TRAILER_LEN)) + .ok_or(ShadowWalErrorV1::LengthOverflow)?; + let mut frame = Vec::with_capacity(total_len); + frame.extend_from_slice(&FRAME_MAGIC.to_be_bytes()); + frame.extend_from_slice(&SHADOW_WAL_FORMAT_VERSION_V1.to_be_bytes()); + frame.extend_from_slice(&0u16.to_be_bytes()); + frame.extend_from_slice(&sequence.to_be_bytes()); + frame.extend_from_slice(&payload_len_u64.to_be_bytes()); + frame.extend_from_slice(&record_count.to_be_bytes()); + frame.extend_from_slice(&0u32.to_be_bytes()); + let header_checksum = crc32c(&frame); + frame.extend_from_slice(&header_checksum.to_be_bytes()); + debug_assert_eq!(frame.len(), FRAME_HEADER_LEN); + for record in records { + let record_len = + u32::try_from(record.len()).expect("record length was validated before encoding"); + frame.extend_from_slice(&record_len.to_be_bytes()); + frame.extend_from_slice(record); + } + let frame_checksum = crc32c(&frame); + frame.extend_from_slice(&frame_checksum.to_be_bytes()); + frame.extend_from_slice(&FRAME_COMMIT_MAGIC.to_be_bytes()); + debug_assert_eq!(frame.len(), total_len); + Ok(frame) +} + +fn validate_frame_header( + header: &[u8; FRAME_HEADER_LEN], + offset: u64, +) -> Result<(), ShadowWalErrorV1> { + if read_u32(&header[0..4]) != FRAME_MAGIC { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "invalid frame magic", + }); + } + if read_u16(&header[4..6]) != SHADOW_WAL_FORMAT_VERSION_V1 { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "unsupported frame version", + }); + } + if read_u16(&header[6..8]) != 0 || read_u32(&header[28..32]) != 0 { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "nonzero frame flags", + }); + } + if read_u32(&header[32..36]) != crc32c(&header[..FRAME_HEADER_PREFIX_LEN]) { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "frame header checksum mismatch", + }); + } + let record_count = read_u32(&header[24..28]); + if record_count == 0 + || usize::try_from(record_count).expect("u32 always fits supported usize") + > MAX_SHADOW_WAL_BATCH_RECORDS_V1 + { + return Err(ShadowWalErrorV1::CorruptFrame { + offset, + reason: "invalid frame record count", + }); + } + Ok(()) +} + +fn decode_records( + payload: &[u8], + declared_records: u32, + frame_offset: u64, +) -> Result>, ShadowWalErrorV1> { + let declared_records = + usize::try_from(declared_records).expect("u32 always fits supported usize"); + let mut records = Vec::with_capacity(declared_records); + let mut cursor = 0usize; + for _ in 0..declared_records { + let length_end = cursor + .checked_add(4) + .ok_or(ShadowWalErrorV1::CorruptFrame { + offset: frame_offset, + reason: "record length offset overflow", + })?; + let Some(length_bytes) = payload.get(cursor..length_end) else { + return Err(ShadowWalErrorV1::CorruptFrame { + offset: frame_offset, + reason: "truncated record length", + }); + }; + let record_len = + usize::try_from(read_u32(length_bytes)).expect("u32 always fits supported usize"); + if record_len > MAX_SHADOW_WAL_RECORD_SIZE_V1 { + return Err(ShadowWalErrorV1::CorruptFrame { + offset: frame_offset, + reason: "record exceeds configured maximum", + }); + } + let record_end = + length_end + .checked_add(record_len) + .ok_or(ShadowWalErrorV1::CorruptFrame { + offset: frame_offset, + reason: "record offset overflow", + })?; + let Some(record) = payload.get(length_end..record_end) else { + return Err(ShadowWalErrorV1::CorruptFrame { + offset: frame_offset, + reason: "truncated record", + }); + }; + records.push(record.to_vec()); + cursor = record_end; + } + if cursor != payload.len() { + return Err(ShadowWalErrorV1::CorruptFrame { + offset: frame_offset, + reason: "trailing frame payload bytes", + }); + } + Ok(records) +} + +fn sync_parent_directory(path: &Path) -> io::Result<()> { + let parent = nonempty_parent(path).unwrap_or_else(|| Path::new(".")); + File::open(parent)?.sync_all() +} + +/// Create a potentially nested WAL parent and durably publish every new +/// directory name from the nearest existing ancestor downwards. +fn create_parent_directories_durable(parent: &Path) -> io::Result<()> { + let mut missing = Vec::new(); + let mut cursor = parent.to_path_buf(); + loop { + if cursor.as_os_str().is_empty() { + cursor = PathBuf::from("."); + } + match fs::symlink_metadata(&cursor) { + Ok(_) => break, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + missing.push(cursor.clone()); + cursor = nonempty_parent(&cursor) + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + } + Err(error) => return Err(error), + } + } + + fs::create_dir_all(parent)?; + for directory in missing.iter().rev() { + File::open(directory)?.sync_all()?; + sync_parent_directory(directory)?; + } + Ok(()) +} + +fn nonempty_parent(path: &Path) -> Option<&Path> { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) +} + +fn read_u16(bytes: &[u8]) -> u16 { + u16::from_be_bytes(bytes.try_into().expect("fixed u16 range")) +} + +fn read_u32(bytes: &[u8]) -> u32 { + u32::from_be_bytes(bytes.try_into().expect("fixed u32 range")) +} + +fn read_u64(bytes: &[u8]) -> u64 { + u64::from_be_bytes(bytes.try_into().expect("fixed u64 range")) +} + +// Table-free CRC32C (Castagnoli). WAL writes are not on the protocol hot path, +// and avoiding another dependency keeps this isolated adapter self-contained. +fn crc32c(bytes: &[u8]) -> u32 { + let mut crc = !0u32; + for byte in bytes { + crc ^= u32::from(*byte); + for _ in 0..8 { + let mask = 0u32.wrapping_sub(crc & 1); + crc = (crc >> 1) ^ (0x82F6_3B78 & mask); + } + } + !crc +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + committee::Committee, starfish_rbc_dag::RbcDagProtocolInstanceId, + types::BlockAuthenticationScheme, + }; + use std::io::{Seek, SeekFrom, Write}; + + fn namespace_with_stakes( + instance_marker: u8, + own_authority: AuthorityIndex, + stakes: Vec, + ) -> ShadowWalNamespaceV1 { + let committee = Committee::new_test(stakes); + let context = RbcDagContextV1::new( + RbcDagProtocolInstanceId::new([instance_marker; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ) + .unwrap(); + ShadowWalNamespaceV1::new(context, own_authority) + } + + fn namespace(instance_marker: u8, own_authority: AuthorityIndex) -> ShadowWalNamespaceV1 { + namespace_with_stakes(instance_marker, own_authority, vec![1; 4]) + } + + fn wal_path(directory: &tempfile::TempDir) -> PathBuf { + directory.path().join("shadow").join("rbc-dag.wal") + } + + fn append_raw(path: &Path, bytes: &[u8]) { + let mut file = OpenOptions::new().append(true).open(path).unwrap(); + file.write_all(bytes).unwrap(); + file.sync_all().unwrap(); + } + + fn overwrite_byte(path: &Path, offset: u64) { + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + file.seek(SeekFrom::Start(offset)).unwrap(); + let mut byte = [0u8; 1]; + file.read_exact(&mut byte).unwrap(); + byte[0] ^= 0x80; + file.seek(SeekFrom::Start(offset)).unwrap(); + file.write_all(&byte).unwrap(); + file.sync_all().unwrap(); + } + + #[test] + fn crc32c_matches_the_standard_check_vector() { + assert_eq!(crc32c(b"123456789"), 0xE306_9283); + } + + #[test] + fn batches_reopen_with_exact_record_bytes_and_continue_sequence() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA1, 2); + let (mut wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.batch_count(), 0); + assert_eq!(recovery.durable_file_len(), FILE_HEADER_LEN as u64); + + let first_records = vec![b"first".to_vec(), Vec::new(), vec![0, 0xFF, 7]]; + let first = wal.append_batch(&first_records).unwrap(); + assert_eq!(first.sequence(), 0); + assert_eq!(first.start_offset(), FILE_HEADER_LEN as u64); + assert_eq!(first.record_count(), 3); + let second_records = vec![b"second".to_vec()]; + let second = wal.append_batch(&second_records).unwrap(); + assert_eq!(second.sequence(), 1); + assert_eq!(second.start_offset(), first.end_offset()); + let summary = wal.shutdown().unwrap(); + assert_eq!(summary.batch_count(), 2); + assert_eq!(summary.record_count(), 4); + + let (mut reopened, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.batch_count(), 2); + assert_eq!(recovery.record_count(), 4); + assert_eq!( + recovery.records(), + vec![ + b"first".as_slice(), + b"".as_slice(), + [0, 0xFF, 7].as_slice(), + b"second".as_slice(), + ] + ); + assert_eq!(recovery.durable_file_len(), summary.file_len()); + assert_eq!( + reopened + .append_batch(&[b"third".to_vec()]) + .unwrap() + .sequence(), + 2 + ); + reopened.shutdown().unwrap(); + + let (_, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.batch_count(), 3); + assert_eq!( + recovery.records().last().copied(), + Some(b"third".as_slice()) + ); + } + + #[test] + fn every_incomplete_final_frame_byte_boundary_is_discarded_once() { + let torn = encode_frame(1, &[b"not-durable".to_vec()]).unwrap(); + for cut in 1..torn.len() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA2, 1); + let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + wal.append_batch(&[b"durable".to_vec()]).unwrap(); + let durable_len = wal.file_len(); + wal.shutdown().unwrap(); + + append_raw(&path, &torn[..cut]); + let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.records(), vec![b"durable".as_slice()]); + assert_eq!(recovery.discarded_tail_bytes(), cut as u64); + assert_eq!(wal.file_len(), durable_len); + wal.shutdown().unwrap(); + + let (_, clean_recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(clean_recovery.discarded_tail_bytes(), 0); + } + } + + #[test] + fn partial_payload_or_trailer_is_discarded_as_one_torn_final_batch() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA3, 0); + let (wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + wal.shutdown().unwrap(); + let frame = encode_frame(0, &[vec![0xAB; 128], b"tail".to_vec()]).unwrap(); + + for cut in [FRAME_HEADER_LEN + 10, frame.len() - 1] { + let original = fs::read(&path).unwrap(); + append_raw(&path, &frame[..cut]); + let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.batch_count(), 0); + assert_eq!(recovery.discarded_tail_bytes(), cut as u64); + wal.shutdown().unwrap(); + fs::write(&path, &original).unwrap(); + } + } + + #[test] + fn checksum_corruption_in_complete_first_or_last_frame_is_rejected() { + for corrupt_first in [true, false] { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA4, 3); + let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + let first = wal.append_batch(&[b"one".to_vec()]).unwrap(); + let second = wal.append_batch(&[b"two".to_vec()]).unwrap(); + wal.shutdown().unwrap(); + let position = if corrupt_first { first } else { second }; + overwrite_byte(&path, position.start_offset() + FRAME_HEADER_LEN as u64 + 4); + assert!(matches!( + ShadowWalV1::open(&path, namespace), + Err(ShadowWalErrorV1::CorruptFrame { + reason: "frame checksum mismatch", + .. + }) + )); + } + } + + #[test] + fn corrupted_frame_header_is_not_misclassified_as_a_torn_tail() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA5, 1); + let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + let frame = wal.append_batch(&[b"record".to_vec()]).unwrap(); + wal.shutdown().unwrap(); + overwrite_byte(&path, frame.start_offset() + 18); + assert!(matches!( + ShadowWalV1::open(&path, namespace), + Err(ShadowWalErrorV1::CorruptFrame { + reason: "frame header checksum mismatch", + .. + }) + )); + } + + #[test] + fn namespace_is_bound_to_instance_committee_authority_and_version() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let expected_namespace = namespace(0xA6, 1); + let (wal, _) = ShadowWalV1::open(&path, expected_namespace).unwrap(); + wal.shutdown().unwrap(); + + assert!(matches!( + ShadowWalV1::open(&path, namespace(0xA7, 1)), + Err(ShadowWalErrorV1::NamespaceMismatch) + )); + assert!(matches!( + ShadowWalV1::open(&path, namespace(0xA6, 2)), + Err(ShadowWalErrorV1::NamespaceMismatch) + )); + assert!(matches!( + ShadowWalV1::open(&path, namespace_with_stakes(0xA6, 1, vec![1, 1, 1, 2])), + Err(ShadowWalErrorV1::NamespaceMismatch) + )); + + let version_offset = 16u64; + overwrite_byte(&path, version_offset + 1); + assert!(matches!( + ShadowWalV1::open(&path, expected_namespace), + Err(ShadowWalErrorV1::UnsupportedFileVersion(_)) + )); + } + + #[test] + fn truncated_or_corrupt_file_header_is_rejected_without_reinitializing() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA8, 0); + let (wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + wal.shutdown().unwrap(); + OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len((FILE_HEADER_LEN - 1) as u64) + .unwrap(); + assert!(matches!( + ShadowWalV1::open(&path, namespace), + Err(ShadowWalErrorV1::TruncatedFileHeader { .. }) + )); + + fs::write(&path, encode_file_header(namespace)).unwrap(); + overwrite_byte(&path, 30); + assert!(matches!( + ShadowWalV1::open(&path, namespace), + Err(ShadowWalErrorV1::InvalidFileHeaderChecksum) + )); + } + + #[test] + fn preexisting_empty_file_is_rejected_instead_of_erasing_durable_identity() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + File::create(&path).unwrap().sync_all().unwrap(); + + assert!(matches!( + ShadowWalV1::open(&path, namespace(0xAB, 0)), + Err(ShadowWalErrorV1::TruncatedFileHeader { + actual: 0, + expected: FILE_HEADER_LEN, + }) + )); + assert_eq!(fs::metadata(path).unwrap().len(), 0); + } + + #[cfg(unix)] + #[test] + fn preexisting_dangling_symlink_fails_closed_without_publication_retry() { + use std::os::unix::fs::symlink; + + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + symlink("missing-shadow-wal", &path).unwrap(); + + assert!(matches!( + ShadowWalV1::open(&path, namespace(0xAF, 0)), + Err(ShadowWalErrorV1::Io(error)) + if error.kind() == io::ErrorKind::NotFound + )); + assert!( + fs::symlink_metadata(&path) + .unwrap() + .file_type() + .is_symlink() + ); + } + + #[test] + fn crash_before_initialization_publish_never_exposes_a_partial_header() { + for cut in [0, 1, FILE_HEADER_LEN - 1, FILE_HEADER_LEN] { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xAC, 0); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + + // Model a process dying at any write boundary before the atomic + // publish. Its private same-directory file may survive, but the + // canonical name must remain absent and independently creatable. + let (temporary_path, mut temporary) = create_initialization_temp(&path).unwrap(); + temporary + .write_all(&encode_file_header(namespace)[..cut]) + .unwrap(); + temporary.sync_all().unwrap(); + drop(temporary); + assert!(!path.exists()); + + let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.durable_file_len(), FILE_HEADER_LEN as u64); + assert_eq!(recovery.batch_count(), 0); + wal.shutdown().unwrap(); + assert_eq!(fs::metadata(&path).unwrap().len(), FILE_HEADER_LEN as u64); + + // Orphan cleanup is opportunistic and never part of identifying + // the canonical WAL. Remove the simulated crashed process's file + // so the test itself leaves no debris. + assert!(temporary_path.exists()); + fs::remove_file(temporary_path).unwrap(); + } + } + + #[test] + fn initialization_race_never_replaces_an_existing_malformed_target() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xAD, 0); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + + let (temporary_path, mut temporary) = create_initialization_temp(&path).unwrap(); + temporary.write_all(&encode_file_header(namespace)).unwrap(); + temporary.sync_all().unwrap(); + File::create(&path).unwrap().sync_all().unwrap(); + + let error = atomic_rename_noreplace(&temporary_path, &path).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + assert!(temporary_path.exists()); + drop(temporary); + fs::remove_file(temporary_path).unwrap(); + + assert!(matches!( + ShadowWalV1::open(&path, namespace), + Err(ShadowWalErrorV1::TruncatedFileHeader { + actual: 0, + expected: FILE_HEADER_LEN, + }) + )); + assert_eq!(fs::metadata(path).unwrap().len(), 0); + } + + #[test] + fn successful_initialization_publishes_only_the_canonical_name() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xAE, 0); + let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.durable_file_len(), FILE_HEADER_LEN as u64); + wal.shutdown().unwrap(); + + let entries = fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert_eq!(entries, vec![path.file_name().unwrap()]); + + let (wal, reopened) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(reopened.discarded_tail_bytes(), 0); + assert_eq!(reopened.durable_file_len(), FILE_HEADER_LEN as u64); + wal.shutdown().unwrap(); + } + + #[test] + fn invalid_batch_is_rejected_without_changing_the_file() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xA9, 0); + let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + let initial_len = wal.file_len(); + assert!(matches!( + wal.append_batch(&[]), + Err(ShadowWalErrorV1::EmptyBatch) + )); + assert!(matches!( + wal.append_batch(&[vec![0; MAX_SHADOW_WAL_RECORD_SIZE_V1 + 1]]), + Err(ShadowWalErrorV1::RecordTooLarge(_)) + )); + assert_eq!(wal.file_len(), initial_len); + assert_eq!(wal.batch_count(), 0); + assert!(!wal.is_poisoned()); + wal.shutdown().unwrap(); + } + + #[test] + fn external_file_mutation_poisoning_requires_reopen() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xAA, 1); + let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + append_raw(&path, &[0xDE, 0xAD]); + assert!(matches!( + wal.append_batch(&[b"event".to_vec()]), + Err(ShadowWalErrorV1::ExternalFileMutation { .. }) + )); + assert!(wal.is_poisoned()); + assert!(matches!( + wal.append_batch(&[b"again".to_vec()]), + Err(ShadowWalErrorV1::Poisoned) + )); + drop(wal); + + let (mut reopened, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!(recovery.discarded_tail_bytes(), 2); + reopened.append_batch(&[b"after-reopen".to_vec()]).unwrap(); + reopened.shutdown().unwrap(); + } +} diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs new file mode 100644 index 00000000..ba10e87f --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -0,0 +1,2659 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Durable, single-owner shadow execution for the embedded-RBC Starfish DAG. +//! +//! This adapter is deliberately non-authoritative: it consumes the same +//! carrier bytes as the live protocol, persists its own deterministic input +//! and trace log, and reports comparison results without influencing the live +//! protocol. One [`ShadowWalV1`] batch is one reducer transition. Effects are +//! returned only after that complete batch has reached durable storage. + +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, + path::Path, +}; + +use crate::{ + crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + starfish_rbc_dag::{ + AuthenticatedCarrierV1, CandidateCarrierV1, CarrierAuthenticationV1, CarrierAuthorizerV1, + CarrierHeaderV1Args, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, + RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, + journal::{ + IngressProvenanceV1, JournalErrorV1, JournalEventV1, ValidatedJournalBatchV1, + WriteAheadJournalV1, + }, + model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, + storage::{ + MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, + ShadowWalSummaryV1, ShadowWalV1, + }, + }, + types::{ + AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, + RoundNumber, TimestampNs, + }, +}; + +const RAW_RECORD_MAGIC: &[u8; 4] = b"SRD3"; +const RAW_RECORD_VERSION_V1: u8 = 1; +const RAW_RECORD_HEADER_SIZE: usize = 80; + +const RECORD_AUTHENTICATED_INGRESS: u8 = 0x01; +const RECORD_CANDIDATE_RETENTION: u8 = 0x02; +const RECORD_CANDIDATE_RECOVERY: u8 = 0x03; +const RECORD_LOCAL_OUTBOUND_CONTENT: u8 = 0x04; +const RECORD_MODEL_TRACE: u8 = 0x10; +const RECORD_LOCAL_OUTBOUND_SIDECAR: u8 = 0x11; +const RECORD_LOCAL_OUTBOUND_EXPOSE: u8 = 0x12; + +const TRACE_ADMISSION_LOCKED: u8 = 0x00; +const TRACE_LOCAL_PHASE_LOCKED: u8 = 0x01; +const TRACE_PHASE_ENTRY_APPLIED: u8 = 0x02; +const TRACE_PHASE_CURSOR_ADVANCED: u8 = 0x03; +const TRACE_LOCAL_CARRIER_FIXED: u8 = 0x04; +const TRACE_DELIVERY_LOCKED: u8 = 0x05; +const TRACE_EFFECT: u8 = 0x06; + +const EFFECT_NEED_CARRIER: u8 = 0x00; +const EFFECT_DELIVERED: u8 = 0x01; +const EFFECT_PREFIX_ADVANCED: u8 = 0x02; +const EFFECT_CARRIER_ROUND_ADVANCED: u8 = 0x03; + +const PHASE_ECHO: u8 = 0x00; +const PHASE_READY: u8 = 0x01; +const PROVENANCE_DIRECT: u8 = 0x00; +const PROVENANCE_RELAYED: u8 = 0x01; + +/// Shadow-benchmark-only resource guard for newly arriving, unsolicited +/// values. This is not a protocol-safe pruning rule: asynchronous delivery +/// can delay an honest INIT by more than this many rounds. A production +/// protocol must derive pruning from a certified/committed watermark instead. +/// Exact RBC recovery requests are exempt from this prototype guard. +const SHADOW_BENCHMARK_UNSOLICITED_RETENTION_WINDOW_ROUNDS_V1: RoundNumber = 64; + +/// Local authentication material owned by exactly one shadow core. +/// +/// The MAC variant contains the local authority's complete pairwise keyring: +/// it creates full outbound vectors and verifies this receiver's inbound tag. +#[derive(Clone, Debug)] +pub(crate) enum ShadowAuthorizerV1 { + Ed25519(Signer), + MlDsa44(MlDsa44Signer), + MlDsa65(MlDsa65Signer), + MacVector(Vec), +} + +impl ShadowAuthorizerV1 { + fn scheme(&self) -> BlockAuthenticationScheme { + match self { + Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, + Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, + Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, + Self::MacVector(_) => BlockAuthenticationScheme::MacVector, + } + } + + fn authorizer(&self, authority: AuthorityIndex) -> CarrierAuthorizerV1<'_> { + match self { + Self::Ed25519(signer) => CarrierAuthorizerV1::Ed25519 { authority, signer }, + Self::MlDsa44(signer) => CarrierAuthorizerV1::MlDsa44 { authority, signer }, + Self::MlDsa65(signer) => CarrierAuthorizerV1::MlDsa65 { authority, signer }, + Self::MacVector(keys) => CarrierAuthorizerV1::MacVector { authority, keys }, + } + } + + fn inbound_mac_keys(&self) -> &[MacKey] { + match self { + Self::MacVector(keys) => keys, + Self::Ed25519(_) | Self::MlDsa44(_) | Self::MlDsa65(_) => &[], + } + } +} + +/// Exact, peer-independent bytes retained for first send and retransmission. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ShadowOutboundEnvelopeV1 { + reference: BlockReference, + canonical_carrier_wire: Vec, + authentication_sidecar: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ShadowIngressDispositionV1 { + Authenticated, + CandidateRetained, + IgnoredDuplicateConflictOrStale, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ShadowIngressOutcomeV1 { + disposition: ShadowIngressDispositionV1, + effects: Vec, +} + +impl ShadowIngressOutcomeV1 { + pub(crate) fn disposition(&self) -> ShadowIngressDispositionV1 { + self.disposition + } + + pub(crate) fn effects(&self) -> &[ModelEffect] { + &self.effects + } + + fn new(disposition: ShadowIngressDispositionV1, effects: Vec) -> Self { + Self { + disposition, + effects, + } + } +} + +impl ShadowOutboundEnvelopeV1 { + pub(crate) fn reference(&self) -> BlockReference { + self.reference + } + + pub(crate) fn canonical_carrier_wire(&self) -> &[u8] { + &self.canonical_carrier_wire + } + + pub(crate) fn authentication_sidecar(&self) -> &[u8] { + &self.authentication_sidecar + } +} + +/// Protocol-independent delivery identity used for direct/shadow comparison. +/// +/// A block reference is intentionally absent: direct and shadow protocols may +/// commit different headers for the same transaction payload. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct ShadowDeliveryIdentityV1 { + pub(crate) author: AuthorityIndex, + pub(crate) round: RoundNumber, + pub(crate) transactions_commitment: TransactionsCommitment, +} + +impl ShadowDeliveryIdentityV1 { + pub(crate) const fn new( + author: AuthorityIndex, + round: RoundNumber, + transactions_commitment: TransactionsCommitment, + ) -> Self { + Self { + author, + round, + transactions_commitment, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct ShadowDeliverySlotV1 { + pub(crate) author: AuthorityIndex, + pub(crate) round: RoundNumber, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ShadowDeliveryComparisonV1 { + Match, + Mismatch { + direct_only: Vec, + shadow_only: Vec, + }, + Ambiguous { + slots: Vec, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ShadowOpenReportV1 { + replayed_batches: u64, + discarded_tail_bytes: u64, + recovery_effects: Vec, +} + +impl ShadowOpenReportV1 { + pub(crate) fn replayed_batches(&self) -> u64 { + self.replayed_batches + } + + pub(crate) fn discarded_tail_bytes(&self) -> u64 { + self.discarded_tail_bytes + } + + /// Final outstanding recovery requests only. Historical delivery, clock, + /// and already-satisfied recovery effects are never reissued on restart. + pub(crate) fn recovery_effects(&self) -> &[ModelEffect] { + &self.recovery_effects + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ShadowCodecErrorV1 { + UnexpectedEnd, + InvalidMagic, + UnsupportedVersion(u8), + InvalidFlags(u16), + ContextMismatch, + AuthorityMismatch { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + AuthenticationSchemeMismatch, + UnknownRecordKind(u8), + InvalidRecordLength(usize), + TrailingBytes(usize), + InvalidProvenance(u8), + InvalidPhase(u8), + InvalidTrace(u8), + InvalidEffect(u8), + NonCanonicalHolders, + LengthOverflow, +} + +impl fmt::Display for ShadowCodecErrorV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "Starfish-RBC-DAG shadow codec error: {self:?}") + } +} + +impl Error for ShadowCodecErrorV1 {} + +#[derive(Debug)] +pub(crate) enum ShadowErrorV1 { + Wal(ShadowWalErrorV1), + Codec(ShadowCodecErrorV1), + Carrier(RbcDagError), + Model(ModelError), + Journal(JournalErrorV1), + ContextMismatch, + UnknownAuthority(AuthorityIndex), + NonCanonicalProvenance, + AuthorizerSchemeMismatch, + AuthorizerKeyMismatch, + InvalidAuthorizerKeyringLength { + expected: usize, + actual: usize, + }, + InvalidBatch(&'static str), + NonCanonicalCarrier, + NonCanonicalAuthentication, + TraceMismatch { + batch_sequence: u64, + }, + ReplayPolicyViolation { + batch_sequence: u64, + reason: &'static str, + }, + UnrequestedRecovery(BlockReference), + SlotCandidateLimit { + author: AuthorityIndex, + round: RoundNumber, + limit: usize, + }, + MissingOutboundCandidate(BlockReference), + MissingDeliveredCandidate(BlockReference), + PostDurabilityCommit(ModelError), + PostDurabilityJournal(JournalErrorV1), + Poisoned, +} + +impl fmt::Display for ShadowErrorV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Wal(error) => write!(formatter, "{error}"), + Self::Codec(error) => write!(formatter, "{error}"), + Self::Carrier(error) => write!(formatter, "{error}"), + Self::Model(error) => write!(formatter, "{error}"), + Self::Journal(error) => write!(formatter, "{error}"), + Self::ContextMismatch => formatter.write_str("shadow protocol context mismatch"), + Self::UnknownAuthority(authority) => { + write!(formatter, "unknown shadow authority {authority}") + } + Self::NonCanonicalProvenance => formatter.write_str( + "non-canonical shadow ingress provenance: an author's own peer must be direct", + ), + Self::AuthorizerSchemeMismatch => { + formatter.write_str("shadow authorizer scheme mismatch") + } + Self::AuthorizerKeyMismatch => formatter.write_str("shadow authorizer key mismatch"), + Self::InvalidAuthorizerKeyringLength { expected, actual } => write!( + formatter, + "invalid shadow authorizer keyring length: expected {expected}, got {actual}" + ), + Self::InvalidBatch(reason) => write!(formatter, "invalid shadow WAL batch: {reason}"), + Self::NonCanonicalCarrier => { + formatter.write_str("non-canonical shadow carrier encoding") + } + Self::NonCanonicalAuthentication => { + formatter.write_str("non-canonical shadow authentication encoding") + } + Self::TraceMismatch { batch_sequence } => write!( + formatter, + "shadow trace mismatch in WAL batch {batch_sequence}" + ), + Self::ReplayPolicyViolation { + batch_sequence, + reason, + } => write!( + formatter, + "shadow replay policy violation in WAL batch {batch_sequence}: {reason}" + ), + Self::UnrequestedRecovery(reference) => { + write!(formatter, "unrequested shadow recovery for {reference}") + } + Self::SlotCandidateLimit { + author, + round, + limit, + } => write!( + formatter, + "shadow candidate limit {limit} reached for slot ({author}, {round})" + ), + Self::MissingOutboundCandidate(reference) => { + write!( + formatter, + "missing persisted outbound shadow candidate {reference}" + ) + } + Self::MissingDeliveredCandidate(reference) => { + write!(formatter, "missing delivered shadow candidate {reference}") + } + Self::PostDurabilityCommit(error) => write!( + formatter, + "shadow model commit failed after WAL durability: {error}" + ), + Self::PostDurabilityJournal(error) => write!( + formatter, + "shadow journal commit failed after WAL durability: {error}" + ), + Self::Poisoned => formatter.write_str("shadow core is poisoned"), + } + } +} + +impl Error for ShadowErrorV1 { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Wal(error) => Some(error), + Self::Codec(error) => Some(error), + Self::Carrier(error) => Some(error), + Self::Model(error) | Self::PostDurabilityCommit(error) => Some(error), + Self::Journal(error) | Self::PostDurabilityJournal(error) => Some(error), + _ => None, + } + } +} + +impl From for ShadowErrorV1 { + fn from(error: ShadowWalErrorV1) -> Self { + Self::Wal(error) + } +} + +impl From for ShadowErrorV1 { + fn from(error: ShadowCodecErrorV1) -> Self { + Self::Codec(error) + } +} + +impl From for ShadowErrorV1 { + fn from(error: RbcDagError) -> Self { + Self::Carrier(error) + } +} + +impl From for ShadowErrorV1 { + fn from(error: ModelError) -> Self { + Self::Model(error) + } +} + +impl From for ShadowErrorV1 { + fn from(error: JournalErrorV1) -> Self { + Self::Journal(error) + } +} + +#[derive(Clone)] +enum ShadowInputV1 { + AuthenticatedIngress { + authenticated: AuthenticatedCarrierV1, + provenance: IngressProvenanceV1, + }, + CandidateRetention(CandidateCarrierV1), + CandidateRecovery(CandidateCarrierV1), + LocalOutbound(LocallyAuthenticatedCarrierV1), +} + +impl ShadowInputV1 { + fn model_input(&self) -> ModelInputRecord { + match self { + Self::AuthenticatedIngress { authenticated, .. } => { + ModelInputRecord::AuthenticatedIngress(authenticated.clone()) + } + Self::CandidateRetention(candidate) => { + ModelInputRecord::CandidateRetained(candidate.clone()) + } + Self::CandidateRecovery(candidate) => { + ModelInputRecord::CandidateRecovered(candidate.clone()) + } + Self::LocalOutbound(authenticated) => { + ModelInputRecord::LocalCarrierFixed(authenticated.clone()) + } + } + } + + fn candidate(&self) -> &CandidateCarrierV1 { + match self { + Self::AuthenticatedIngress { authenticated, .. } => authenticated.candidate(), + Self::CandidateRetention(candidate) | Self::CandidateRecovery(candidate) => candidate, + Self::LocalOutbound(authenticated) => authenticated.candidate(), + } + } + + fn is_local(&self) -> bool { + matches!(self, Self::LocalOutbound(_)) + } +} + +struct DecodedRawRecord { + kind: u8, + payload: Vec, +} + +/// Synchronous, non-authoritative shadow core. +/// +/// This type has one mutable model, journal, and WAL handle and intentionally +/// offers no shared-state wrapper. A caller may move it between threads but +/// must preserve exclusive ownership. +pub(crate) struct StarfishRbcDagShadowV1 { + committee: RbcDagCommitteeContextV1, + context: RbcDagContextV1, + own_authority: AuthorityIndex, + authorizer: ShadowAuthorizerV1, + model: RbcDagModel, + journal: WriteAheadJournalV1, + wal: ShadowWalV1, + candidates: BTreeMap, + delivered: BTreeSet, + authenticated_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, + ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, + slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, + requested_recoveries: BTreeMap>, + poisoned: bool, +} + +impl StarfishRbcDagShadowV1 { + pub(crate) fn open( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { + validate_configuration(&committee, own_authority, context, &authorizer)?; + let namespace = ShadowWalNamespaceV1::new(context, own_authority); + let (wal, recovery) = ShadowWalV1::open(path, namespace)?; + let replayed_batches = recovery.batch_count(); + let discarded_tail_bytes = recovery.discarded_tail_bytes(); + + let model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; + let journal = WriteAheadJournalV1::new(context, own_authority); + let mut core = Self { + committee, + context, + own_authority, + authorizer, + model, + journal, + wal, + candidates: BTreeMap::new(), + delivered: BTreeSet::new(), + authenticated_slots: BTreeMap::new(), + ordinarily_retained_slots: BTreeMap::new(), + slot_candidates: BTreeMap::new(), + requested_recoveries: BTreeMap::new(), + poisoned: false, + }; + + for batch in recovery.batches() { + let input = core.decode_batch(batch.records())?; + core.apply_replayed(input, batch.records(), batch.sequence())?; + } + let recovery_effects = core + .requested_recoveries + .iter() + .map(|(target, holders)| ModelEffect::NeedCarrier { + target: *target, + holders: holders.clone(), + }) + .collect(); + Ok(( + core, + ShadowOpenReportV1 { + replayed_batches, + discarded_tail_bytes, + recovery_effects, + }, + )) + } + + pub(crate) fn local_carrier_round(&self) -> RoundNumber { + self.model.local_carrier_round() + } + + pub(crate) fn can_create_carrier(&self) -> bool { + self.model.can_create_carrier() + } + + pub(crate) fn wal_counts(&self) -> (u64, u64) { + (self.wal.batch_count(), self.wal.record_count()) + } + + #[cfg(test)] + pub(crate) fn delivered( + &self, + authority: AuthorityIndex, + round: RoundNumber, + ) -> Option { + self.model.delivered(authority, round) + } + + /// Construct, authenticate, durably fix, and expose the next local + /// carrier. M3 intentionally uses empty ACKs and no consensus vertex. + pub(crate) fn create_local_carrier( + &mut self, + round: RoundNumber, + transactions_commitment: TransactionsCommitment, + creation_time_ns: TimestampNs, + ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { + self.ensure_live()?; + let (own_prev, weak_parents) = self.model.local_parent_set()?; + let candidate = CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author: self.own_authority, + carrier_round: round, + own_prev, + weak_parents, + transactions_commitment, + data_acknowledgments: Vec::new(), + phase_batch: self.model.pending_phase_batch(), + consensus_vertex: None, + creation_time_ns, + }, + &self.committee, + )?; + let authenticated = self.context.authenticate_local_with_committee( + candidate, + &self.committee, + self.authorizer.authorizer(self.own_authority), + )?; + let envelope = ShadowOutboundEnvelopeV1 { + reference: authenticated.reference(), + canonical_carrier_wire: authenticated.candidate().canonical_wire_bytes()?, + authentication_sidecar: authenticated.authentication().canonical_wire_bytes(), + }; + let effects = self.apply_durable(ShadowInputV1::LocalOutbound(authenticated))?; + debug_assert!( + self.journal + .snapshot() + .outbound(envelope.reference()) + .is_some_and(|outbound| outbound.exposed()) + ); + Ok((envelope, effects)) + } + + /// Verify and durably apply an authenticated network envelope for this + /// exact receiver. + #[cfg(test)] + pub(crate) fn receive_authenticated_envelope( + &mut self, + canonical_carrier_wire: &[u8], + authentication_sidecar: &[u8], + provenance: IngressProvenanceV1, + ) -> Result, ShadowErrorV1> { + self.ensure_live()?; + let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; + validate_provenance(provenance, candidate.header().author(), &self.committee)?; + self.receive_decoded_authenticated(candidate, authentication_sidecar, provenance) + } + + /// Classify provenance from the authenticated transport peer and the + /// already-decoded carrier author, then verify and apply the envelope. + /// The candidate bytes are decoded exactly once in this method. + #[cfg(test)] + pub(crate) fn receive_authenticated_from_peer( + &mut self, + canonical_carrier_wire: &[u8], + authentication_sidecar: &[u8], + trusted_peer: AuthorityIndex, + ) -> Result, ShadowErrorV1> { + self.ensure_live()?; + if !self.committee.committee().known_authority(trusted_peer) { + return Err(ShadowErrorV1::UnknownAuthority(trusted_peer)); + } + let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; + let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); + self.receive_decoded_authenticated(candidate, authentication_sidecar, provenance) + } + + /// Verify an envelope when possible and otherwise durably retain its + /// canonical content as candidate-only input. This is the normal network + /// ingress API: a poisoned receiver tag must not discard the content that + /// embedded ECHO/READY evidence can later deliver. + pub(crate) fn receive_or_retain_from_peer( + &mut self, + canonical_carrier_wire: &[u8], + authentication_sidecar: &[u8], + trusted_peer: AuthorityIndex, + ) -> Result { + self.ensure_live()?; + if !self.committee.committee().known_authority(trusted_peer) { + return Err(ShadowErrorV1::UnknownAuthority(trusted_peer)); + } + let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; + let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); + // Once a slot has a durably authenticated value, unsolicited replays + // and conflicts cannot change the shadow state. Reject before public + // signature/ML-DSA verification to keep this idempotence cheap. + if self.ignores_unsolicited_authenticated(&candidate) { + return Ok(ShadowIngressOutcomeV1::new( + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale, + Vec::new(), + )); + } + match self.authenticate_decoded(candidate.clone(), authentication_sidecar) { + Ok(authenticated) => { + let (effects, applied) = + self.apply_authenticated_capability(authenticated, provenance)?; + let disposition = if applied { + ShadowIngressDispositionV1::Authenticated + } else { + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale + }; + Ok(ShadowIngressOutcomeV1::new(disposition, effects)) + } + Err(ShadowErrorV1::Carrier(_)) | Err(ShadowErrorV1::NonCanonicalAuthentication) => { + let (effects, applied) = self.apply_candidate_retention(candidate)?; + let disposition = if applied { + ShadowIngressDispositionV1::CandidateRetained + } else { + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale + }; + Ok(ShadowIngressOutcomeV1::new(disposition, effects)) + } + Err(error) => Err(error), + } + } + + #[cfg(test)] + fn receive_decoded_authenticated( + &mut self, + candidate: CandidateCarrierV1, + authentication_sidecar: &[u8], + provenance: IngressProvenanceV1, + ) -> Result, ShadowErrorV1> { + let authenticated = self.authenticate_decoded(candidate, authentication_sidecar)?; + self.apply_authenticated_capability(authenticated, provenance) + .map(|(effects, _)| effects) + } + + fn authenticate_decoded( + &self, + candidate: CandidateCarrierV1, + authentication_sidecar: &[u8], + ) -> Result { + let authentication = decode_authentication(authentication_sidecar, &self.committee)?; + Ok(self.context.verify_authentication_with_committee( + candidate, + authentication, + self.own_authority, + &self.committee, + self.authorizer.inbound_mac_keys(), + )?) + } + + fn apply_authenticated_capability( + &mut self, + authenticated: AuthenticatedCarrierV1, + provenance: IngressProvenanceV1, + ) -> Result<(Vec, bool), ShadowErrorV1> { + let reference = authenticated.reference(); + let slot = carrier_slot(reference); + if round_is_stale(self.model.local_carrier_round(), reference.round) + || self.authenticated_slots.contains_key(&slot) + { + return Ok((Vec::new(), false)); + } + self.apply_durable(ShadowInputV1::AuthenticatedIngress { + authenticated, + provenance, + }) + .map(|effects| (effects, true)) + } + + fn ignores_unsolicited_authenticated(&self, candidate: &CandidateCarrierV1) -> bool { + let reference = candidate.reference(); + round_is_stale(self.model.local_carrier_round(), reference.round) + || self + .authenticated_slots + .contains_key(&carrier_slot(reference)) + } + + fn apply_candidate_retention( + &mut self, + candidate: CandidateCarrierV1, + ) -> Result<(Vec, bool), ShadowErrorV1> { + let reference = candidate.reference(); + let slot = carrier_slot(reference); + if round_is_stale(self.model.local_carrier_round(), reference.round) + || self.candidates.contains_key(&reference) + || self.authenticated_slots.contains_key(&slot) + || self.ordinarily_retained_slots.contains_key(&slot) + { + return Ok((Vec::new(), false)); + } + self.apply_durable(ShadowInputV1::CandidateRetention(candidate)) + .map(|effects| (effects, true)) + } + + /// Recover content only if it recomputes the exact requested reference. + pub(crate) fn recover_candidate_for( + &mut self, + expected_reference: BlockReference, + canonical_carrier_wire: &[u8], + ) -> Result, ShadowErrorV1> { + self.ensure_live()?; + let candidate = decode_candidate( + canonical_carrier_wire, + &self.committee, + Some(expected_reference), + )?; + self.apply_requested_recovery(candidate) + } + + pub(crate) fn retained_candidate_wire(&self, reference: BlockReference) -> Option> { + self.journal + .snapshot() + .retained_carrier(reference) + .map(<[u8]>::to_vec) + } + + /// Return every exposed local carrier in deterministic reference order. + /// The same full sidecar is returned for every peer. + pub(crate) fn retransmissions(&self) -> Vec { + self.journal + .snapshot() + .retransmissions() + .into_iter() + .map(|outbound| ShadowOutboundEnvelopeV1 { + reference: outbound.reference(), + canonical_carrier_wire: outbound.canonical_carrier_wire().to_vec(), + authentication_sidecar: outbound.authentication_sidecar().to_vec(), + }) + .collect() + } + + /// Metadata for every durably exposed local carrier, ordered by its exact + /// persisted reference. Startup uses this to prove that the shadow WAL and + /// recovered authoritative local chain overlap on the same application + /// payload and creation timestamp before accepting new observations. + pub(crate) fn local_outbound_metadata( + &self, + ) -> Result, ShadowErrorV1> { + self.journal + .snapshot() + .retransmissions() + .into_iter() + .map(|outbound| { + let reference = outbound.reference(); + let candidate = self + .candidates + .get(&reference) + .ok_or(ShadowErrorV1::MissingOutboundCandidate(reference))?; + Ok(( + candidate.header().carrier_round(), + candidate.header().transactions_commitment(), + candidate.header().creation_time_ns(), + )) + }) + .collect() + } + + pub(crate) fn delivered_identities( + &self, + ) -> Result, ShadowErrorV1> { + self.delivered + .iter() + .map(|reference| { + let candidate = self + .candidates + .get(reference) + .ok_or(ShadowErrorV1::MissingDeliveredCandidate(*reference))?; + Ok(ShadowDeliveryIdentityV1::new( + candidate.header().author(), + candidate.header().carrier_round(), + candidate.header().transactions_commitment(), + )) + }) + .collect() + } + + /// Compare protocol-independent delivery sets. Multiple transaction + /// commitments for one `(author, round)` slot make the comparison + /// ambiguous instead of being resolved by arrival or reference order. + #[cfg(test)] + pub(crate) fn compare_direct_deliveries( + &self, + direct: I, + ) -> Result + where + I: IntoIterator, + { + let direct: BTreeSet<_> = direct.into_iter().collect(); + let shadow: BTreeSet<_> = self.delivered_identities()?.into_iter().collect(); + let ambiguous = ambiguous_slots(&direct) + .into_iter() + .chain(ambiguous_slots(&shadow)) + .collect::>(); + if !ambiguous.is_empty() { + return Ok(ShadowDeliveryComparisonV1::Ambiguous { + slots: ambiguous.into_iter().collect(), + }); + } + if direct == shadow { + return Ok(ShadowDeliveryComparisonV1::Match); + } + Ok(ShadowDeliveryComparisonV1::Mismatch { + direct_only: direct.difference(&shadow).copied().collect(), + shadow_only: shadow.difference(&direct).copied().collect(), + }) + } + + pub(crate) fn shutdown(self) -> Result { + Ok(self.wal.shutdown()?) + } + + fn ensure_live(&self) -> Result<(), ShadowErrorV1> { + if self.poisoned || self.wal.is_poisoned() { + Err(ShadowErrorV1::Poisoned) + } else { + Ok(()) + } + } + + fn apply_requested_recovery( + &mut self, + candidate: CandidateCarrierV1, + ) -> Result, ShadowErrorV1> { + let reference = candidate.reference(); + if self.candidates.contains_key(&reference) { + return Ok(Vec::new()); + } + if !self.requested_recoveries.contains_key(&reference) { + return Err(ShadowErrorV1::UnrequestedRecovery(reference)); + } + let slot = carrier_slot(reference); + let limit = self + .committee + .committee() + .len() + .saturating_mul(2) + .saturating_add(2); + if self + .slot_candidates + .get(&slot) + .is_some_and(|candidates| candidates.len() >= limit) + { + return Err(ShadowErrorV1::SlotCandidateLimit { + author: reference.authority, + round: reference.round, + limit, + }); + } + self.apply_durable(ShadowInputV1::CandidateRecovery(candidate)) + } + + fn apply_durable(&mut self, input: ShadowInputV1) -> Result, ShadowErrorV1> { + let plan = self.model.plan_input(input.model_input())?; + let records = encode_batch(self.context, self.own_authority, &input, plan.trace())?; + let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; + self.wal.append_batch(&records)?; + let effects = match self.model.commit_plan(plan) { + Ok(effects) => effects, + Err(error) => { + self.poisoned = true; + return Err(ShadowErrorV1::PostDurabilityCommit(error)); + } + }; + if let Err(error) = self.journal.commit_validated_batch(journal_batch) { + self.poisoned = true; + return Err(ShadowErrorV1::PostDurabilityJournal(error)); + } + self.record_committed_input(&input, &effects); + Ok(effects) + } + + fn apply_replayed( + &mut self, + input: ShadowInputV1, + records: &[Vec], + batch_sequence: u64, + ) -> Result, ShadowErrorV1> { + self.validate_replay_policy(&input, batch_sequence)?; + let recorded_trace = decode_recorded_trace( + records, + self.context, + self.own_authority, + self.committee.committee().len(), + )?; + let plan = self.model.plan_input(input.model_input())?; + if plan.trace() != recorded_trace { + return Err(ShadowErrorV1::TraceMismatch { batch_sequence }); + } + let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; + let effects = self.model.commit_plan(plan)?; + if let Err(error) = self.journal.commit_validated_batch(journal_batch) { + self.poisoned = true; + return Err(ShadowErrorV1::PostDurabilityJournal(error)); + } + self.record_committed_input(&input, &effects); + Ok(effects) + } + + fn validate_replay_policy( + &self, + input: &ShadowInputV1, + batch_sequence: u64, + ) -> Result<(), ShadowErrorV1> { + let reference = input.candidate().reference(); + let slot = carrier_slot(reference); + let violation = match input { + ShadowInputV1::AuthenticatedIngress { .. } => { + if round_is_stale(self.model.local_carrier_round(), reference.round) { + Some("stale authenticated ingress") + } else if self.authenticated_slots.contains_key(&slot) { + Some("duplicate or conflicting authenticated slot") + } else { + None + } + } + ShadowInputV1::CandidateRetention(_) => { + if round_is_stale(self.model.local_carrier_round(), reference.round) { + Some("stale candidate retention") + } else if self.candidates.contains_key(&reference) + || self.authenticated_slots.contains_key(&slot) + || self.ordinarily_retained_slots.contains_key(&slot) + { + Some("duplicate or conflicting ordinary retention") + } else { + None + } + } + ShadowInputV1::CandidateRecovery(_) => { + let limit = self + .committee + .committee() + .len() + .saturating_mul(2) + .saturating_add(2); + if self.candidates.contains_key(&reference) { + Some("duplicate candidate recovery") + } else if !self.requested_recoveries.contains_key(&reference) { + Some("candidate recovery was not requested") + } else if self + .slot_candidates + .get(&slot) + .is_some_and(|candidates| candidates.len() >= limit) + { + Some("candidate recovery exceeds the per-slot limit") + } else { + None + } + } + ShadowInputV1::LocalOutbound(_) => None, + }; + if let Some(reason) = violation { + return Err(ShadowErrorV1::ReplayPolicyViolation { + batch_sequence, + reason, + }); + } + Ok(()) + } + + fn record_committed_input(&mut self, input: &ShadowInputV1, effects: &[ModelEffect]) { + let candidate = input.candidate().clone(); + let reference = candidate.reference(); + let slot = carrier_slot(reference); + self.candidates.insert(reference, candidate); + self.slot_candidates + .entry(slot) + .or_default() + .insert(reference); + match input { + ShadowInputV1::AuthenticatedIngress { .. } | ShadowInputV1::LocalOutbound(_) => { + self.authenticated_slots.entry(slot).or_insert(reference); + } + ShadowInputV1::CandidateRetention(_) => { + self.ordinarily_retained_slots + .entry(slot) + .or_insert(reference); + } + ShadowInputV1::CandidateRecovery(_) => {} + } + self.requested_recoveries.remove(&reference); + for effect in effects { + match effect { + ModelEffect::NeedCarrier { target, holders } => { + self.requested_recoveries.insert(*target, holders.clone()); + } + ModelEffect::Delivered(delivered) => { + self.delivered.insert(*delivered); + self.requested_recoveries.remove(delivered); + } + ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} + } + } + } + + fn decode_batch(&self, records: &[Vec]) -> Result { + if records.is_empty() { + return Err(ShadowErrorV1::InvalidBatch("empty transition batch")); + } + let decoded = records + .iter() + .map(|record| { + decode_raw_record(record, self.context, self.own_authority) + .map_err(ShadowErrorV1::from) + }) + .collect::, _>>()?; + match decoded[0].kind { + RECORD_AUTHENTICATED_INGRESS => { + ensure_trace_tail(&decoded[1..])?; + let mut payload = RawDecoder::new(&decoded[0].payload); + let provenance = decode_provenance(&mut payload)?; + let carrier_wire = payload.read_sized_bytes()?.to_vec(); + let sidecar_wire = payload.read_sized_bytes()?.to_vec(); + payload.finish()?; + let candidate = decode_candidate(&carrier_wire, &self.committee, None)?; + validate_provenance(provenance, candidate.header().author(), &self.committee)?; + let authentication = decode_authentication(&sidecar_wire, &self.committee)?; + let authenticated = self.context.verify_authentication_with_committee( + candidate, + authentication, + self.own_authority, + &self.committee, + self.authorizer.inbound_mac_keys(), + )?; + Ok(ShadowInputV1::AuthenticatedIngress { + authenticated, + provenance, + }) + } + RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY => { + ensure_trace_tail(&decoded[1..])?; + let candidate = decode_candidate(&decoded[0].payload, &self.committee, None)?; + if decoded[0].kind == RECORD_CANDIDATE_RETENTION { + Ok(ShadowInputV1::CandidateRetention(candidate)) + } else { + Ok(ShadowInputV1::CandidateRecovery(candidate)) + } + } + RECORD_LOCAL_OUTBOUND_CONTENT => { + if decoded.len() != 4 { + return Err(ShadowErrorV1::InvalidBatch( + "local transition must contain input, trace, sidecar, and exposure", + )); + } + let trace_end = decoded.len() - 2; + ensure_trace_tail(&decoded[1..trace_end])?; + if decoded[trace_end].kind != RECORD_LOCAL_OUTBOUND_SIDECAR + || decoded[trace_end + 1].kind != RECORD_LOCAL_OUTBOUND_EXPOSE + { + return Err(ShadowErrorV1::InvalidBatch( + "local sidecar and exposure must follow the trace", + )); + } + let candidate = decode_candidate(&decoded[0].payload, &self.committee, None)?; + let mut sidecar = RawDecoder::new(&decoded[trace_end].payload); + let sidecar_reference = sidecar.read_reference()?; + let authentication_wire = sidecar.read_sized_bytes()?.to_vec(); + sidecar.finish()?; + let mut expose = RawDecoder::new(&decoded[trace_end + 1].payload); + let expose_reference = expose.read_reference()?; + expose.finish()?; + if sidecar_reference != candidate.reference() + || expose_reference != candidate.reference() + { + return Err(ShadowErrorV1::InvalidBatch( + "local sidecar or exposure reference mismatch", + )); + } + let authentication = decode_authentication(&authentication_wire, &self.committee)?; + let authenticated = self.context.verify_local_authentication_with_committee( + candidate, + authentication, + &self.committee, + self.authorizer.authorizer(self.own_authority), + )?; + Ok(ShadowInputV1::LocalOutbound(authenticated)) + } + _ => Err(ShadowErrorV1::InvalidBatch( + "first record is not a model input", + )), + } + } +} + +fn validate_configuration( + committee: &RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: &ShadowAuthorizerV1, +) -> Result<(), ShadowErrorV1> { + if context.committee_id() != committee.committee_id() { + return Err(ShadowErrorV1::ContextMismatch); + } + if !committee.committee().known_authority(own_authority) { + return Err(ShadowErrorV1::UnknownAuthority(own_authority)); + } + if context.authentication_scheme() != authorizer.scheme() { + return Err(ShadowErrorV1::AuthorizerSchemeMismatch); + } + match authorizer { + ShadowAuthorizerV1::Ed25519(signer) => { + if committee.committee().get_public_key(own_authority) != Some(&signer.public_key()) { + return Err(ShadowErrorV1::AuthorizerKeyMismatch); + } + } + ShadowAuthorizerV1::MlDsa44(signer) => { + if committee + .committee() + .get_ml_dsa_44_public_key(own_authority) + != Some(&signer.public_key()) + { + return Err(ShadowErrorV1::AuthorizerKeyMismatch); + } + } + ShadowAuthorizerV1::MlDsa65(signer) => { + if committee + .committee() + .get_ml_dsa_65_public_key(own_authority) + != Some(&signer.public_key()) + { + return Err(ShadowErrorV1::AuthorizerKeyMismatch); + } + } + ShadowAuthorizerV1::MacVector(keys) => { + if keys.len() != committee.committee().len() { + return Err(ShadowErrorV1::InvalidAuthorizerKeyringLength { + expected: committee.committee().len(), + actual: keys.len(), + }); + } + } + } + Ok(()) +} + +fn validate_provenance( + provenance: IngressProvenanceV1, + candidate_author: AuthorityIndex, + committee: &RbcDagCommitteeContextV1, +) -> Result<(), ShadowErrorV1> { + if let IngressProvenanceV1::Relayed { peer } = provenance { + if !committee.committee().known_authority(peer) { + return Err(ShadowErrorV1::UnknownAuthority(peer)); + } + if peer == candidate_author { + return Err(ShadowErrorV1::NonCanonicalProvenance); + } + } + Ok(()) +} + +pub(crate) fn infer_ingress_provenance( + trusted_peer: AuthorityIndex, + candidate_author: AuthorityIndex, +) -> IngressProvenanceV1 { + if trusted_peer == candidate_author { + IngressProvenanceV1::DirectFromAuthor + } else { + IngressProvenanceV1::Relayed { peer: trusted_peer } + } +} + +fn decode_candidate( + wire: &[u8], + committee: &RbcDagCommitteeContextV1, + expected_reference: Option, +) -> Result { + let candidate = + CandidateCarrierV1::decode_wire_with_committee(wire, committee, expected_reference)?; + if candidate.canonical_wire_bytes()?.as_slice() != wire { + return Err(ShadowErrorV1::NonCanonicalCarrier); + } + Ok(candidate) +} + +fn decode_authentication( + wire: &[u8], + committee: &RbcDagCommitteeContextV1, +) -> Result { + let authentication = CarrierAuthenticationV1::decode_wire_with_committee(wire, committee)?; + if authentication.canonical_wire_bytes().as_slice() != wire { + return Err(ShadowErrorV1::NonCanonicalAuthentication); + } + Ok(authentication) +} + +fn validate_journal_transition( + journal: &WriteAheadJournalV1, + input: &ShadowInputV1, + trace: &[ModelTraceEvent], +) -> Result { + let mut events = Vec::new(); + let context = journal.snapshot().context(); + match input { + ShadowInputV1::AuthenticatedIngress { + authenticated, + provenance, + } => events.push(JournalEventV1::AuthenticatedIngress { + context, + sequence: journal.snapshot().next_ingress_sequence(), + authenticated: authenticated.clone(), + provenance: *provenance, + }), + ShadowInputV1::CandidateRetention(candidate) + | ShadowInputV1::CandidateRecovery(candidate) => { + events.push(JournalEventV1::RetainCandidateContent { + context, + candidate: candidate.clone(), + }); + } + ShadowInputV1::LocalOutbound(authenticated) => { + events.push(JournalEventV1::PersistOutboundContent { + context, + candidate: authenticated.candidate().clone(), + }); + } + } + + let local_reference = input.is_local().then(|| input.candidate().reference()); + for entry in trace { + let event = match entry { + ModelTraceEvent::AdmissionLocked(target) if Some(*target) == local_reference => { + // `FixOwnCarrier` is the journal's authority lock for a local + // value. `LockAdmission` intentionally accepts only network + // ingress; the exact model trace remains present in the WAL. + None + } + ModelTraceEvent::AdmissionLocked(target) => Some(JournalEventV1::LockAdmission { + context, + target: *target, + }), + ModelTraceEvent::LocalPhaseLocked(statement) => Some(match statement { + RbcPhaseStatementV1::Echo { target } => JournalEventV1::LockEcho { + context, + target: *target, + }, + RbcPhaseStatementV1::Ready { target } => JournalEventV1::LockReady { + context, + target: *target, + }, + }), + ModelTraceEvent::PhaseBatchEntryApplied { + outer, + index, + sender, + statement, + } => Some(JournalEventV1::ApplyPhaseStatement { + context, + outer: *outer, + index: *index, + sender: *sender, + statement: *statement, + }), + ModelTraceEvent::PhaseBatchCursorAdvanced { + outer, + index, + next_index, + } => { + if index.checked_add(1) != Some(*next_index) { + return Err(ShadowErrorV1::InvalidBatch( + "non-sequential phase cursor trace", + )); + } + Some(JournalEventV1::AdvancePhaseBatchCursor { + context, + outer: *outer, + index: *index, + }) + } + ModelTraceEvent::LocalCarrierFixed(reference) => Some(JournalEventV1::FixOwnCarrier { + context, + reference: *reference, + }), + ModelTraceEvent::DeliveryLocked(target) => Some(JournalEventV1::LockDelivery { + context, + target: *target, + }), + ModelTraceEvent::Effect(_) => None, + }; + if let Some(event) = event { + events.push(event); + } + } + + if let ShadowInputV1::LocalOutbound(authenticated) = input { + events.push(JournalEventV1::PersistOutboundSidecar { + context, + authenticated: authenticated.clone(), + }); + events.push(JournalEventV1::ExposeOutbound { + context, + reference: authenticated.reference(), + }); + } + journal.validate_batch(events).map_err(Into::into) +} + +fn encode_batch( + context: RbcDagContextV1, + own_authority: AuthorityIndex, + input: &ShadowInputV1, + trace: &[ModelTraceEvent], +) -> Result>, ShadowErrorV1> { + let mut records = Vec::with_capacity(4); + match input { + ShadowInputV1::AuthenticatedIngress { + authenticated, + provenance, + } => { + let mut payload = Vec::new(); + encode_provenance(&mut payload, *provenance); + push_sized_bytes( + &mut payload, + &authenticated.candidate().canonical_wire_bytes()?, + )?; + push_sized_bytes( + &mut payload, + &authenticated.authentication().canonical_wire_bytes(), + )?; + records.push(encode_raw_record( + context, + own_authority, + RECORD_AUTHENTICATED_INGRESS, + &payload, + )?); + } + ShadowInputV1::CandidateRetention(candidate) => { + records.push(encode_raw_record( + context, + own_authority, + RECORD_CANDIDATE_RETENTION, + &candidate.canonical_wire_bytes()?, + )?); + } + ShadowInputV1::CandidateRecovery(candidate) => { + records.push(encode_raw_record( + context, + own_authority, + RECORD_CANDIDATE_RECOVERY, + &candidate.canonical_wire_bytes()?, + )?); + } + ShadowInputV1::LocalOutbound(authenticated) => { + records.push(encode_raw_record( + context, + own_authority, + RECORD_LOCAL_OUTBOUND_CONTENT, + &authenticated.candidate().canonical_wire_bytes()?, + )?); + } + } + records.push(encode_raw_record( + context, + own_authority, + RECORD_MODEL_TRACE, + &encode_trace_batch(trace)?, + )?); + if let ShadowInputV1::LocalOutbound(authenticated) = input { + let mut sidecar = Vec::new(); + push_reference(&mut sidecar, authenticated.reference()); + push_sized_bytes( + &mut sidecar, + &authenticated.authentication().canonical_wire_bytes(), + )?; + records.push(encode_raw_record( + context, + own_authority, + RECORD_LOCAL_OUTBOUND_SIDECAR, + &sidecar, + )?); + let mut expose = Vec::new(); + push_reference(&mut expose, authenticated.reference()); + records.push(encode_raw_record( + context, + own_authority, + RECORD_LOCAL_OUTBOUND_EXPOSE, + &expose, + )?); + } + Ok(records) +} + +fn encode_raw_record( + context: RbcDagContextV1, + own_authority: AuthorityIndex, + kind: u8, + payload: &[u8], +) -> Result, ShadowCodecErrorV1> { + let payload_len = + u32::try_from(payload.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; + let total_len = RAW_RECORD_HEADER_SIZE + .checked_add(payload.len()) + .ok_or(ShadowCodecErrorV1::LengthOverflow)?; + if total_len > MAX_SHADOW_WAL_RECORD_SIZE_V1 { + return Err(ShadowCodecErrorV1::InvalidRecordLength(total_len)); + } + let mut bytes = Vec::with_capacity(total_len); + bytes.extend_from_slice(RAW_RECORD_MAGIC); + bytes.push(RAW_RECORD_VERSION_V1); + bytes.push(kind); + bytes.extend_from_slice(&0u16.to_be_bytes()); + bytes.extend_from_slice(context.protocol_instance().as_bytes()); + bytes.extend_from_slice(context.committee_id().as_bytes()); + bytes.extend_from_slice(&own_authority.to_be_bytes()); + bytes.push(authentication_scheme_code(context.authentication_scheme())); + bytes.push(0); + bytes.extend_from_slice(&payload_len.to_be_bytes()); + debug_assert_eq!(bytes.len(), RAW_RECORD_HEADER_SIZE); + bytes.extend_from_slice(payload); + Ok(bytes) +} + +fn decode_raw_record( + bytes: &[u8], + context: RbcDagContextV1, + own_authority: AuthorityIndex, +) -> Result { + if bytes.len() > MAX_SHADOW_WAL_RECORD_SIZE_V1 { + return Err(ShadowCodecErrorV1::InvalidRecordLength(bytes.len())); + } + let mut decoder = RawDecoder::new(bytes); + if decoder.take(4)? != RAW_RECORD_MAGIC { + return Err(ShadowCodecErrorV1::InvalidMagic); + } + let version = decoder.read_u8()?; + if version != RAW_RECORD_VERSION_V1 { + return Err(ShadowCodecErrorV1::UnsupportedVersion(version)); + } + let kind = decoder.read_u8()?; + let flags = decoder.read_u16()?; + if flags != 0 { + return Err(ShadowCodecErrorV1::InvalidFlags(flags)); + } + if decoder.take(32)? != context.protocol_instance().as_bytes() + || decoder.take(32)? != context.committee_id().as_bytes() + { + return Err(ShadowCodecErrorV1::ContextMismatch); + } + let actual_authority = decoder.read_u16()?; + if actual_authority != own_authority { + return Err(ShadowCodecErrorV1::AuthorityMismatch { + expected: own_authority, + actual: actual_authority, + }); + } + if decoder.read_u8()? != authentication_scheme_code(context.authentication_scheme()) { + return Err(ShadowCodecErrorV1::AuthenticationSchemeMismatch); + } + if decoder.read_u8()? != 0 { + return Err(ShadowCodecErrorV1::InvalidFlags(1)); + } + let payload_len = decoder.read_u32()? as usize; + if payload_len != decoder.remaining() { + return Err(ShadowCodecErrorV1::InvalidRecordLength(bytes.len())); + } + let payload = decoder.take(payload_len)?.to_vec(); + decoder.finish()?; + match kind { + RECORD_AUTHENTICATED_INGRESS + | RECORD_CANDIDATE_RETENTION + | RECORD_CANDIDATE_RECOVERY + | RECORD_LOCAL_OUTBOUND_CONTENT + | RECORD_MODEL_TRACE + | RECORD_LOCAL_OUTBOUND_SIDECAR + | RECORD_LOCAL_OUTBOUND_EXPOSE => {} + other => return Err(ShadowCodecErrorV1::UnknownRecordKind(other)), + } + Ok(DecodedRawRecord { kind, payload }) +} + +fn decode_recorded_trace( + records: &[Vec], + context: RbcDagContextV1, + own_authority: AuthorityIndex, + committee_size: usize, +) -> Result, ShadowErrorV1> { + let decoded = records + .iter() + .map(|record| decode_raw_record(record, context, own_authority)) + .collect::, _>>()?; + let range = match decoded.first().map(|record| record.kind) { + Some(RECORD_LOCAL_OUTBOUND_CONTENT) => 1..decoded.len().saturating_sub(2), + Some( + RECORD_AUTHENTICATED_INGRESS | RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY, + ) => 1..decoded.len(), + _ => return Err(ShadowErrorV1::InvalidBatch("missing model input")), + }; + let trace_records = &decoded[range]; + ensure_trace_tail(trace_records)?; + decode_trace_batch(&trace_records[0].payload, committee_size).map_err(ShadowErrorV1::from) +} + +fn ensure_trace_tail(records: &[DecodedRawRecord]) -> Result<(), ShadowErrorV1> { + if matches!(records, [record] if record.kind == RECORD_MODEL_TRACE) { + Ok(()) + } else { + Err(ShadowErrorV1::InvalidBatch( + "each model input must have exactly one ordered trace record", + )) + } +} + +fn encode_trace_batch(trace: &[ModelTraceEvent]) -> Result, ShadowCodecErrorV1> { + let count = u32::try_from(trace.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; + let mut bytes = Vec::new(); + bytes.extend_from_slice(&count.to_be_bytes()); + for entry in trace { + push_sized_bytes(&mut bytes, &encode_trace(entry)?)?; + } + Ok(bytes) +} + +fn decode_trace_batch( + bytes: &[u8], + committee_size: usize, +) -> Result, ShadowCodecErrorV1> { + let mut decoder = RawDecoder::new(bytes); + let count = decoder.read_u32()? as usize; + if count > decoder.remaining() / 5 { + return Err(ShadowCodecErrorV1::InvalidRecordLength(bytes.len())); + } + let mut trace = Vec::with_capacity(count); + for _ in 0..count { + trace.push(decode_trace(decoder.read_sized_bytes()?, committee_size)?); + } + decoder.finish()?; + Ok(trace) +} + +fn encode_trace(trace: &ModelTraceEvent) -> Result, ShadowCodecErrorV1> { + let mut bytes = Vec::new(); + match trace { + ModelTraceEvent::AdmissionLocked(reference) => { + bytes.push(TRACE_ADMISSION_LOCKED); + push_reference(&mut bytes, *reference); + } + ModelTraceEvent::LocalPhaseLocked(statement) => { + bytes.push(TRACE_LOCAL_PHASE_LOCKED); + push_phase(&mut bytes, *statement); + } + ModelTraceEvent::PhaseBatchEntryApplied { + outer, + index, + sender, + statement, + } => { + bytes.push(TRACE_PHASE_ENTRY_APPLIED); + push_reference(&mut bytes, *outer); + push_usize_as_u32(&mut bytes, *index)?; + bytes.extend_from_slice(&sender.to_be_bytes()); + push_phase(&mut bytes, *statement); + } + ModelTraceEvent::PhaseBatchCursorAdvanced { + outer, + index, + next_index, + } => { + bytes.push(TRACE_PHASE_CURSOR_ADVANCED); + push_reference(&mut bytes, *outer); + push_usize_as_u32(&mut bytes, *index)?; + push_usize_as_u32(&mut bytes, *next_index)?; + } + ModelTraceEvent::LocalCarrierFixed(reference) => { + bytes.push(TRACE_LOCAL_CARRIER_FIXED); + push_reference(&mut bytes, *reference); + } + ModelTraceEvent::DeliveryLocked(reference) => { + bytes.push(TRACE_DELIVERY_LOCKED); + push_reference(&mut bytes, *reference); + } + ModelTraceEvent::Effect(effect) => { + bytes.push(TRACE_EFFECT); + push_effect(&mut bytes, effect)?; + } + } + Ok(bytes) +} + +fn decode_trace( + bytes: &[u8], + committee_size: usize, +) -> Result { + let mut decoder = RawDecoder::new(bytes); + let trace = match decoder.read_u8()? { + TRACE_ADMISSION_LOCKED => ModelTraceEvent::AdmissionLocked(decoder.read_reference()?), + TRACE_LOCAL_PHASE_LOCKED => ModelTraceEvent::LocalPhaseLocked(decoder.read_phase()?), + TRACE_PHASE_ENTRY_APPLIED => ModelTraceEvent::PhaseBatchEntryApplied { + outer: decoder.read_reference()?, + index: decoder.read_u32()? as usize, + sender: decoder.read_u16()?, + statement: decoder.read_phase()?, + }, + TRACE_PHASE_CURSOR_ADVANCED => ModelTraceEvent::PhaseBatchCursorAdvanced { + outer: decoder.read_reference()?, + index: decoder.read_u32()? as usize, + next_index: decoder.read_u32()? as usize, + }, + TRACE_LOCAL_CARRIER_FIXED => ModelTraceEvent::LocalCarrierFixed(decoder.read_reference()?), + TRACE_DELIVERY_LOCKED => ModelTraceEvent::DeliveryLocked(decoder.read_reference()?), + TRACE_EFFECT => ModelTraceEvent::Effect(decoder.read_effect(committee_size)?), + other => return Err(ShadowCodecErrorV1::InvalidTrace(other)), + }; + decoder.finish()?; + Ok(trace) +} + +fn push_effect(bytes: &mut Vec, effect: &ModelEffect) -> Result<(), ShadowCodecErrorV1> { + match effect { + ModelEffect::NeedCarrier { target, holders } => { + bytes.push(EFFECT_NEED_CARRIER); + push_reference(bytes, *target); + let count = + u16::try_from(holders.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; + bytes.extend_from_slice(&count.to_be_bytes()); + for holder in holders { + bytes.extend_from_slice(&holder.to_be_bytes()); + } + } + ModelEffect::Delivered(reference) => { + bytes.push(EFFECT_DELIVERED); + push_reference(bytes, *reference); + } + ModelEffect::PrefixAdvanced { authority, tip } => { + bytes.push(EFFECT_PREFIX_ADVANCED); + bytes.extend_from_slice(&authority.to_be_bytes()); + push_reference(bytes, *tip); + } + ModelEffect::CarrierRoundAdvanced(round) => { + bytes.push(EFFECT_CARRIER_ROUND_ADVANCED); + bytes.extend_from_slice(&round.to_be_bytes()); + } + } + Ok(()) +} + +fn push_phase(bytes: &mut Vec, statement: RbcPhaseStatementV1) { + match statement { + RbcPhaseStatementV1::Echo { target } => { + bytes.push(PHASE_ECHO); + push_reference(bytes, target); + } + RbcPhaseStatementV1::Ready { target } => { + bytes.push(PHASE_READY); + push_reference(bytes, target); + } + } +} + +fn encode_provenance(bytes: &mut Vec, provenance: IngressProvenanceV1) { + match provenance { + IngressProvenanceV1::DirectFromAuthor => bytes.push(PROVENANCE_DIRECT), + IngressProvenanceV1::Relayed { peer } => { + bytes.push(PROVENANCE_RELAYED); + bytes.extend_from_slice(&peer.to_be_bytes()); + } + } +} + +fn decode_provenance( + decoder: &mut RawDecoder<'_>, +) -> Result { + match decoder.read_u8()? { + PROVENANCE_DIRECT => Ok(IngressProvenanceV1::DirectFromAuthor), + PROVENANCE_RELAYED => Ok(IngressProvenanceV1::Relayed { + peer: decoder.read_u16()?, + }), + other => Err(ShadowCodecErrorV1::InvalidProvenance(other)), + } +} + +fn push_reference(bytes: &mut Vec, reference: BlockReference) { + bytes.extend_from_slice(&reference.authority.to_be_bytes()); + bytes.extend_from_slice(&reference.round.to_be_bytes()); + bytes.extend_from_slice(reference.digest.as_ref()); +} + +fn push_sized_bytes(target: &mut Vec, bytes: &[u8]) -> Result<(), ShadowCodecErrorV1> { + let len = u32::try_from(bytes.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; + target.extend_from_slice(&len.to_be_bytes()); + target.extend_from_slice(bytes); + Ok(()) +} + +fn push_usize_as_u32(target: &mut Vec, value: usize) -> Result<(), ShadowCodecErrorV1> { + let value = u32::try_from(value).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; + target.extend_from_slice(&value.to_be_bytes()); + Ok(()) +} + +fn authentication_scheme_code(scheme: BlockAuthenticationScheme) -> u8 { + match scheme { + BlockAuthenticationScheme::Ed25519 => 0, + BlockAuthenticationScheme::MlDsa44 => 1, + BlockAuthenticationScheme::MlDsa65 => 2, + BlockAuthenticationScheme::MacVector => 3, + } +} + +fn carrier_slot(reference: BlockReference) -> (AuthorityIndex, RoundNumber) { + (reference.authority, reference.round) +} + +fn round_is_stale(current_round: RoundNumber, candidate_round: RoundNumber) -> bool { + candidate_round + < current_round.saturating_sub(SHADOW_BENCHMARK_UNSOLICITED_RETENTION_WINDOW_ROUNDS_V1) +} + +#[cfg(test)] +fn ambiguous_slots( + identities: &BTreeSet, +) -> BTreeSet { + let mut by_slot: BTreeMap> = + BTreeMap::new(); + for identity in identities { + by_slot + .entry(ShadowDeliverySlotV1 { + author: identity.author, + round: identity.round, + }) + .or_default() + .insert(identity.transactions_commitment); + } + by_slot + .into_iter() + .filter_map(|(slot, commitments)| (commitments.len() > 1).then_some(slot)) + .collect() +} + +struct RawDecoder<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> RawDecoder<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + fn remaining(&self) -> usize { + self.bytes.len().saturating_sub(self.position) + } + + fn take(&mut self, length: usize) -> Result<&'a [u8], ShadowCodecErrorV1> { + let end = self + .position + .checked_add(length) + .ok_or(ShadowCodecErrorV1::LengthOverflow)?; + let bytes = self + .bytes + .get(self.position..end) + .ok_or(ShadowCodecErrorV1::UnexpectedEnd)?; + self.position = end; + Ok(bytes) + } + + fn read_u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn read_u16(&mut self) -> Result { + Ok(u16::from_be_bytes( + self.take(2)?.try_into().expect("fixed u16 range"), + )) + } + + fn read_u32(&mut self) -> Result { + Ok(u32::from_be_bytes( + self.take(4)?.try_into().expect("fixed u32 range"), + )) + } + + fn read_reference(&mut self) -> Result { + let authority = self.read_u16()?; + let round = self.read_u32()?; + let mut digest = [0; 32]; + digest.copy_from_slice(self.take(32)?); + Ok(BlockReference { + authority, + round, + digest: BlockDigest::from(digest), + }) + } + + fn read_phase(&mut self) -> Result { + let phase = self.read_u8()?; + let target = self.read_reference()?; + match phase { + PHASE_ECHO => Ok(RbcPhaseStatementV1::Echo { target }), + PHASE_READY => Ok(RbcPhaseStatementV1::Ready { target }), + other => Err(ShadowCodecErrorV1::InvalidPhase(other)), + } + } + + fn read_effect(&mut self, committee_size: usize) -> Result { + match self.read_u8()? { + EFFECT_NEED_CARRIER => { + let target = self.read_reference()?; + let count = self.read_u16()? as usize; + if count > committee_size || count > MAX_COMMITTEE_SIZE as usize { + return Err(ShadowCodecErrorV1::NonCanonicalHolders); + } + let mut holders = Vec::with_capacity(count); + let mut previous = None; + for _ in 0..count { + let holder = self.read_u16()?; + if holder as usize >= committee_size + || previous.is_some_and(|previous| previous >= holder) + { + return Err(ShadowCodecErrorV1::NonCanonicalHolders); + } + previous = Some(holder); + holders.push(holder); + } + Ok(ModelEffect::NeedCarrier { target, holders }) + } + EFFECT_DELIVERED => Ok(ModelEffect::Delivered(self.read_reference()?)), + EFFECT_PREFIX_ADVANCED => Ok(ModelEffect::PrefixAdvanced { + authority: self.read_u16()?, + tip: self.read_reference()?, + }), + EFFECT_CARRIER_ROUND_ADVANCED => { + Ok(ModelEffect::CarrierRoundAdvanced(self.read_u32()?)) + } + other => Err(ShadowCodecErrorV1::InvalidEffect(other)), + } + } + + fn read_sized_bytes(&mut self) -> Result<&'a [u8], ShadowCodecErrorV1> { + let length = self.read_u32()? as usize; + self.take(length) + } + + fn finish(self) -> Result<(), ShadowCodecErrorV1> { + if self.position == self.bytes.len() { + Ok(()) + } else { + Err(ShadowCodecErrorV1::TrailingBytes( + self.bytes.len() - self.position, + )) + } + } +} + +const _: () = assert!(RAW_RECORD_HEADER_SIZE == 80); + +#[cfg(test)] +mod tests { + use super::*; + use std::{fs::OpenOptions, io::Write, sync::Arc}; + + use tempfile::TempDir; + + use crate::{ + committee::Committee, + crypto::{ + MAC_TAG_SIZE, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, + mac_keyrings_for_test, + }, + starfish_rbc_dag::{RbcDagProtocolInstanceId, carrier_genesis_reference}, + }; + + const N: usize = 4; + + struct TestNetwork { + committee: RbcDagCommitteeContextV1, + context: RbcDagContextV1, + keyrings: Vec>, + directories: Vec, + nodes: Vec, + } + + impl TestNetwork { + fn new() -> Self { + let committee = Committee::new_test(vec![1; N]); + let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xE3; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ); + let keyrings = mac_keyrings_for_test(N); + let directories = (0..N) + .map(|_| tempfile::tempdir().unwrap()) + .collect::>(); + let nodes = (0..N) + .map(|authority| { + StarfishRbcDagShadowV1::open( + directories[authority].path().join("shadow.wal"), + committee.clone(), + authority as AuthorityIndex, + context, + ShadowAuthorizerV1::MacVector(keyrings[authority].clone()), + ) + .unwrap() + .0 + }) + .collect(); + Self { + committee, + context, + keyrings, + directories, + nodes, + } + } + + fn path(&self, authority: usize) -> std::path::PathBuf { + self.directories[authority].path().join("shadow.wal") + } + + fn run_three_rounds_with_one_poisoned_recipient(&mut self) { + for round in 1..=3 { + let envelopes = self + .nodes + .iter_mut() + .enumerate() + .map(|(authority, node)| { + let commitment = TransactionsCommitment::from_bytes( + [(round * 16 + authority as u32) as u8; 32], + ); + node.create_local_carrier(round, commitment, u64::from(round) * 1_000) + .unwrap() + .0 + }) + .collect::>(); + + for (sender, envelope) in envelopes.iter().enumerate() { + for receiver in 0..N { + if receiver == sender { + continue; + } + if round == 1 && sender == 0 && receiver == 2 { + let mut poisoned = envelope.authentication_sidecar().to_vec(); + poisoned[3 + receiver * MAC_TAG_SIZE] ^= 1; + let outcome = self.nodes[receiver] + .receive_or_retain_from_peer( + envelope.canonical_carrier_wire(), + &poisoned, + sender as AuthorityIndex, + ) + .unwrap(); + assert_eq!( + outcome.disposition(), + ShadowIngressDispositionV1::CandidateRetained + ); + assert!(outcome.effects().is_empty()); + } else { + self.nodes[receiver] + .receive_authenticated_from_peer( + envelope.canonical_carrier_wire(), + envelope.authentication_sidecar(), + sender as AuthorityIndex, + ) + .unwrap(); + } + } + } + + for node in &self.nodes { + assert_eq!(node.local_carrier_round(), round + 1); + } + } + } + } + + #[test] + fn three_round_mac_shadow_delivers_round_one_after_poisoned_tag_is_only_staged() { + let mut network = TestNetwork::new(); + network.run_three_rounds_with_one_poisoned_recipient(); + + for node in &network.nodes { + for author in 0..N { + assert!(node.delivered(author as AuthorityIndex, 1).is_some()); + } + } + assert!(network.nodes[2].delivered(0, 1).is_some()); + assert_eq!( + infer_ingress_provenance(1, 1), + IngressProvenanceV1::DirectFromAuthor + ); + assert_eq!( + infer_ingress_provenance(2, 1), + IngressProvenanceV1::Relayed { peer: 2 } + ); + } + + #[test] + fn authenticated_replays_and_slot_conflicts_do_not_grow_durable_state() { + let mut network = TestNetwork::new(); + let first = round_one_candidate(1, &network.committee, 0x71); + let first_authentication = network + .context + .authenticate_with_committee( + &first, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + let wire = first.canonical_wire_bytes().unwrap(); + let sidecar = first_authentication.canonical_wire_bytes(); + let accepted = network.nodes[0] + .receive_or_retain_from_peer(&wire, &sidecar, 1) + .unwrap(); + assert_eq!( + accepted.disposition(), + ShadowIngressDispositionV1::Authenticated + ); + let counts = network.nodes[0].wal_counts(); + + let replay = network.nodes[0] + .receive_or_retain_from_peer(&wire, &sidecar, 1) + .unwrap(); + assert_eq!( + replay.disposition(), + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale + ); + assert_eq!(network.nodes[0].wal_counts(), counts); + + let conflicting = round_one_candidate(1, &network.committee, 0x72); + let conflicting_authentication = network + .context + .authenticate_with_committee( + &conflicting, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + let conflict = network.nodes[0] + .receive_or_retain_from_peer( + &conflicting.canonical_wire_bytes().unwrap(), + &conflicting_authentication.canonical_wire_bytes(), + 1, + ) + .unwrap(); + assert_eq!( + conflict.disposition(), + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale + ); + assert_eq!(network.nodes[0].wal_counts(), counts); + + // The classified runtime path cheaply ignores an occupied slot, but + // the explicitly strict API must retain its advertised verification + // contract even when the candidate cannot affect state. + let mut invalid_conflict_sidecar = conflicting_authentication.canonical_wire_bytes(); + invalid_conflict_sidecar[3] ^= 1; + assert!(matches!( + network.nodes[0].receive_authenticated_from_peer( + &conflicting.canonical_wire_bytes().unwrap(), + &invalid_conflict_sidecar, + 1, + ), + Err(ShadowErrorV1::Carrier(_)) + )); + assert_eq!(network.nodes[0].wal_counts(), counts); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let (mut restarted, _) = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap(); + assert_eq!(restarted.wal_counts(), counts); + let replay_after_restart = restarted + .receive_or_retain_from_peer(&wire, &sidecar, 1) + .unwrap(); + assert_eq!( + replay_after_restart.disposition(), + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale + ); + assert_eq!(restarted.wal_counts(), counts); + } + + #[test] + fn unsolicited_stale_window_has_a_fixed_round_bound() { + assert!(!round_is_stale(64, 0)); + assert!(!round_is_stale(65, 1)); + assert!(round_is_stale(66, 1)); + assert!(!round_is_stale(66, 2)); + } + + #[test] + fn caller_supplied_relay_provenance_must_be_canonical_for_the_author() { + let mut network = TestNetwork::new(); + let candidate = round_one_candidate(1, &network.committee, 0x80); + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + assert!(matches!( + network.nodes[0].receive_authenticated_envelope( + &candidate.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + IngressProvenanceV1::Relayed { peer: 1 }, + ), + Err(ShadowErrorV1::NonCanonicalProvenance) + )); + assert_eq!(network.nodes[0].wal_counts(), (0, 0)); + } + + #[test] + fn replay_rejects_a_duplicate_authenticated_slot_even_with_an_exact_trace() { + let mut network = TestNetwork::new(); + let candidate = round_one_candidate(1, &network.committee, 0x81); + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &candidate.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 1, + ) + .unwrap(); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let namespace = ShadowWalNamespaceV1::new(network.context, 0); + let (mut wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + let duplicate = recovery.batches()[0].records().to_vec(); + wal.append_batch(&duplicate).unwrap(); + wal.shutdown().unwrap(); + + let result = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ); + assert!(matches!( + result, + Err(ShadowErrorV1::ReplayPolicyViolation { + batch_sequence: 1, + reason: "duplicate or conflicting authenticated slot", + }) + )); + } + + #[test] + fn replay_rejects_candidate_recovery_that_was_never_requested() { + let mut network = TestNetwork::new(); + let target = round_one_candidate(3, &network.committee, 0x82); + let outer = round_two_phase_carrier( + 1, + RbcPhaseStatementV1::Echo { + target: target.reference(), + }, + &network.committee, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 1, + ) + .unwrap(); + assert!( + !network.nodes[0] + .requested_recoveries + .contains_key(&target.reference()) + ); + + // The reducer accepts content after any phase evidence allocated the + // candidate. The shadow's durable grammar is deliberately stricter: + // network recovery is legal only after a surfaced NeedCarrier. + let forged_input = ShadowInputV1::CandidateRecovery(target); + let plan = network.nodes[0] + .model + .plan_input(forged_input.model_input()) + .unwrap(); + let forged_batch = encode_batch(network.context, 0, &forged_input, plan.trace()).unwrap(); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let namespace = ShadowWalNamespaceV1::new(network.context, 0); + let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); + wal.append_batch(&forged_batch).unwrap(); + wal.shutdown().unwrap(); + + let result = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ); + assert!(matches!( + result, + Err(ShadowErrorV1::ReplayPolicyViolation { + batch_sequence: 1, + reason: "candidate recovery was not requested", + }) + )); + } + + #[test] + fn wal_restart_past_round_one_discards_torn_tail_and_retransmits_exact_bytes() { + let mut network = TestNetwork::new(); + network.run_three_rounds_with_one_poisoned_recipient(); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + let before = node.retransmissions(); + assert_eq!(before.len(), 3); + let retained = node.retained_candidate_wire(before[0].reference()).unwrap(); + assert_eq!(retained, before[0].canonical_carrier_wire()); + node.shutdown().unwrap(); + + let namespace = ShadowWalNamespaceV1::new(network.context, 0); + let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + let first = recovery.batches().first().unwrap().records(); + let kinds = first + .iter() + .map(|record| decode_raw_record(record, network.context, 0).unwrap().kind) + .collect::>(); + assert_eq!(kinds[0], RECORD_LOCAL_OUTBOUND_CONTENT); + assert_eq!(kinds[kinds.len() - 2], RECORD_LOCAL_OUTBOUND_SIDECAR); + assert_eq!(kinds[kinds.len() - 1], RECORD_LOCAL_OUTBOUND_EXPOSE); + assert!( + kinds[1..kinds.len() - 2] + .iter() + .all(|kind| *kind == RECORD_MODEL_TRACE) + ); + wal.shutdown().unwrap(); + + let torn = b"uncommitted-tail"; + let mut file = OpenOptions::new().append(true).open(&path).unwrap(); + file.write_all(torn).unwrap(); + file.sync_all().unwrap(); + drop(file); + + let (restarted, report) = StarfishRbcDagShadowV1::open( + &path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap(); + assert_eq!(report.discarded_tail_bytes(), torn.len() as u64); + assert!(report.replayed_batches() > 3); + assert_eq!(restarted.local_carrier_round(), 4); + assert_eq!(restarted.retransmissions(), before); + assert!(report.recovery_effects().is_empty()); + for author in 0..N { + assert!(restarted.delivered(author as AuthorityIndex, 1).is_some()); + } + } + + #[test] + fn every_authentication_scheme_reopens_with_the_exact_persisted_sidecar() { + let committee = Committee::new_test(vec![1; N]); + let committee = RbcDagCommitteeContextV1::new(committee).unwrap(); + let keyrings = mac_keyrings_for_test(N); + let directory = tempfile::tempdir().unwrap(); + + for (index, scheme) in [ + BlockAuthenticationScheme::Ed25519, + BlockAuthenticationScheme::MlDsa44, + BlockAuthenticationScheme::MlDsa65, + BlockAuthenticationScheme::MacVector, + ] + .into_iter() + .enumerate() + { + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xA0 + index as u8; 32]).unwrap(), + &committee, + scheme, + ); + let authorizer = match scheme { + BlockAuthenticationScheme::Ed25519 => ShadowAuthorizerV1::Ed25519(dummy_signer()), + BlockAuthenticationScheme::MlDsa44 => { + ShadowAuthorizerV1::MlDsa44(dummy_ml_dsa_44_signer()) + } + BlockAuthenticationScheme::MlDsa65 => { + ShadowAuthorizerV1::MlDsa65(dummy_ml_dsa_65_signer()) + } + BlockAuthenticationScheme::MacVector => { + ShadowAuthorizerV1::MacVector(keyrings[0].clone()) + } + }; + let path = directory.path().join(format!("scheme-{index}.wal")); + let (mut core, _) = StarfishRbcDagShadowV1::open( + &path, + committee.clone(), + 0, + context, + authorizer.clone(), + ) + .unwrap(); + let envelope = core + .create_local_carrier( + 1, + TransactionsCommitment::from_bytes([0xE0 + index as u8; 32]), + 10, + ) + .unwrap() + .0; + core.shutdown().unwrap(); + + let (restarted, report) = + StarfishRbcDagShadowV1::open(&path, committee.clone(), 0, context, authorizer) + .unwrap(); + assert!(report.replayed_batches() > 0); + assert_eq!(restarted.retransmissions(), vec![envelope]); + restarted.shutdown().unwrap(); + } + } + + #[test] + fn reopening_rejects_a_wrong_local_signer_before_replaying_the_wal() { + let committee = Committee::new_test(vec![1; N]); + let committee = RbcDagCommitteeContextV1::new(committee).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xAF; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::Ed25519, + ); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("wrong-reopen-signer.wal"); + let (mut core, _) = StarfishRbcDagShadowV1::open( + &path, + committee.clone(), + 0, + context, + ShadowAuthorizerV1::Ed25519(dummy_signer()), + ) + .unwrap(); + core.create_local_carrier(1, TransactionsCommitment::from_bytes([0xEF; 32]), 10) + .unwrap(); + core.shutdown().unwrap(); + + let wrong_signer = Signer::new_for_test(1).pop().unwrap(); + let result = StarfishRbcDagShadowV1::open( + path, + committee, + 0, + context, + ShadowAuthorizerV1::Ed25519(wrong_signer), + ); + assert!(matches!(result, Err(ShadowErrorV1::AuthorizerKeyMismatch))); + } + + #[test] + fn replay_rejects_a_validly_framed_corrupt_trace() { + let committee = Committee::new_test(vec![1; N]); + let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xF4; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ); + let keyrings = mac_keyrings_for_test(N); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("corrupt-trace.wal"); + let candidate = round_one_candidate(1, &committee, 0x91); + let input = encode_raw_record( + context, + 0, + RECORD_CANDIDATE_RETENTION, + &candidate.canonical_wire_bytes().unwrap(), + ) + .unwrap(); + let fabricated_trace = encode_raw_record( + context, + 0, + RECORD_MODEL_TRACE, + &encode_trace_batch(&[ModelTraceEvent::Effect(ModelEffect::CarrierRoundAdvanced( + 99, + ))]) + .unwrap(), + ) + .unwrap(); + let (mut wal, _) = ShadowWalV1::open(&path, ShadowWalNamespaceV1::new(context, 0)).unwrap(); + wal.append_batch(&[input, fabricated_trace]).unwrap(); + wal.shutdown().unwrap(); + + let result = StarfishRbcDagShadowV1::open( + &path, + committee, + 0, + context, + ShadowAuthorizerV1::MacVector(keyrings[0].clone()), + ); + assert!(matches!( + result, + Err(ShadowErrorV1::TraceMismatch { batch_sequence: 0 }) + )); + } + + #[test] + fn direct_shadow_comparison_reports_match_mismatch_and_ambiguity_without_references() { + let mut network = TestNetwork::new(); + network.run_three_rounds_with_one_poisoned_recipient(); + let node = &network.nodes[0]; + let direct = node.delivered_identities().unwrap(); + assert_eq!( + node.compare_direct_deliveries(direct.clone()).unwrap(), + ShadowDeliveryComparisonV1::Match + ); + + let mut mismatch = direct.clone(); + mismatch[0].transactions_commitment = TransactionsCommitment::from_bytes([0xAB; 32]); + assert!(matches!( + node.compare_direct_deliveries(mismatch).unwrap(), + ShadowDeliveryComparisonV1::Mismatch { .. } + )); + + let mut ambiguous = direct.clone(); + let mut conflicting = direct[0]; + conflicting.transactions_commitment = TransactionsCommitment::from_bytes([0xCD; 32]); + ambiguous.push(conflicting); + assert_eq!( + node.compare_direct_deliveries(ambiguous).unwrap(), + ShadowDeliveryComparisonV1::Ambiguous { + slots: vec![ShadowDeliverySlotV1 { + author: direct[0].author, + round: direct[0].round, + }], + } + ); + } + + #[test] + fn recovery_api_binds_requested_reference_before_model_transition() { + let mut network = TestNetwork::new(); + let candidate = round_one_candidate(1, &network.committee, 0xA1); + let mut wrong = candidate.reference(); + wrong.digest = BlockDigest::from([0xFF; 32]); + assert!(matches!( + network.nodes[0] + .recover_candidate_for(wrong, &candidate.canonical_wire_bytes().unwrap()), + Err(ShadowErrorV1::Carrier( + RbcDagError::ReferenceMismatch { .. } + )) + )); + } + + #[test] + fn recovered_content_is_durable_only_after_embedded_phase_evidence() { + let mut network = TestNetwork::new(); + let target = round_one_candidate(3, &network.committee, 0xB1); + for sender in 1..N { + let outer = round_two_phase_carrier( + sender as AuthorityIndex, + RbcPhaseStatementV1::Echo { + target: target.reference(), + }, + &network.committee, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: sender as AuthorityIndex, + keys: &network.keyrings[sender], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + sender as AuthorityIndex, + ) + .unwrap(); + } + assert!( + network.nodes[0] + .retained_candidate_wire(target.reference()) + .is_none() + ); + network.nodes[0] + .recover_candidate_for(target.reference(), &target.canonical_wire_bytes().unwrap()) + .unwrap(); + assert_eq!( + network.nodes[0] + .retained_candidate_wire(target.reference()) + .unwrap(), + target.canonical_wire_bytes().unwrap() + ); + } + + fn round_one_candidate( + author: AuthorityIndex, + committee: &RbcDagCommitteeContextV1, + marker: u8, + ) -> CandidateCarrierV1 { + let weak_parents = committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(carrier_genesis_reference) + .collect(); + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 1, + own_prev: carrier_genesis_reference(author), + weak_parents, + transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: 1, + }, + committee, + ) + .unwrap() + } + + fn round_two_phase_carrier( + author: AuthorityIndex, + statement: RbcPhaseStatementV1, + committee: &RbcDagCommitteeContextV1, + ) -> CandidateCarrierV1 { + let previous = |authority: AuthorityIndex| BlockReference { + authority, + round: 1, + digest: BlockDigest::from([0xC0 + authority as u8; 32]), + }; + let weak_parents = committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(previous) + .collect(); + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 2, + own_prev: previous(author), + weak_parents, + transactions_commitment: TransactionsCommitment::from_bytes( + [0xD0 + author as u8; 32], + ), + data_acknowledgments: Vec::new(), + phase_batch: vec![statement], + consensus_vertex: None, + creation_time_ns: 2, + }, + committee, + ) + .unwrap() + } +} diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs new file mode 100644 index 00000000..49879cc2 --- /dev/null +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -0,0 +1,2071 @@ +// Copyright (c) 2026 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Async, non-authoritative network adapter for the persisted RBC-DAG shadow. + +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, + path::Path, + sync::Arc, + time::{Duration, Instant}, +}; + +use parking_lot::Mutex; + +use tokio::{ + sync::{ + mpsc::{self, error::TrySendError}, + oneshot, + }, + task::JoinHandle, +}; + +use crate::{ + crypto::{MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE}, + network::{NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse}, + starfish_rbc::RbcCanonicalHeader, + starfish_rbc_dag::{ + MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, + model::{ModelEffect, ModelError}, + }, + starfish_rbc_dag_shadow::{ + ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, + ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, ShadowOpenReportV1, + ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, + }, + types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, +}; + +// A shadow run must absorb one complete committee fan-in plus a small reserve +// for the local carrier and control notifications before its single fsync +// owner can drain. At the four-MiB carrier cap, allowing at most 64 queued +// inputs also caps carrier payload retention at 256 MiB (plus bounded +// sidecars and allocator overhead). Larger committees are rejected for this +// benchmark prototype instead of silently under-sizing the queue and +// reporting incomparable results. +// Use the full bounded allowance even for a small committee. A single fan-in +// reserve is insufficient when several round bursts arrive while the actor is +// synchronously making the previous transition durable. +const SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1: usize = 64; +const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 64; +const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; +const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; +const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ShadowLocalCarrierV1 { + author: AuthorityIndex, + round: RoundNumber, + transactions_commitment: crate::crypto::TransactionsCommitment, + creation_time_ns: TimestampNs, +} + +impl ShadowLocalCarrierV1 { + fn from_direct_header(header: &RbcCanonicalHeader) -> Self { + Self { + author: header.reference().authority, + round: header.reference().round, + transactions_commitment: header.transactions_commitment(), + creation_time_ns: header.meta_creation_time_ns(), + } + } +} + +enum ShadowServiceMessageV1 { + LocalCarrier(ShadowLocalCarrierV1), + Carrier { + peer: AuthorityIndex, + envelope: RbcDagShadowCarrier, + }, + CarrierRequest { + peer: AuthorityIndex, + reference: BlockReference, + }, + CarrierResponse { + peer: AuthorityIndex, + response: RbcDagShadowCarrierResponse, + }, + DirectDeliveriesChanged, + TopologyChanged, + RetryRecovery, + Shutdown(oneshot::Sender>), +} + +#[derive(Clone)] +pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { + sender: mpsc::Sender, + max_sidecar_size: usize, + own_authority: AuthorityIndex, + committee_size: usize, + input_capacity: usize, + desired_topology: Arc>>, + desired_direct_deliveries: Arc>>, + invalidated_by_overload: Arc>>, +} + +impl StarfishRbcDagShadowServiceHandleV1 { + fn send(&self, message: ShadowServiceMessageV1) -> Result<(), ShadowServiceErrorV1> { + let kind = message.kind(); + if let Some(reason) = *self.invalidated_by_overload.lock() { + return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); + } + self.sender.try_send(message).map_err(|error| match error { + TrySendError::Full(_) => { + *self.invalidated_by_overload.lock() = Some(kind); + ShadowServiceErrorV1::Overloaded { + kind, + capacity: self.input_capacity, + } + } + TrySendError::Closed(_) => ShadowServiceErrorV1::Stopped, + }) + } + + pub(crate) fn local_header( + &self, + header: &RbcCanonicalHeader, + ) -> Result<(), ShadowServiceErrorV1> { + self.send(ShadowServiceMessageV1::LocalCarrier( + ShadowLocalCarrierV1::from_direct_header(header), + )) + } + + pub(crate) fn carrier( + &self, + peer: AuthorityIndex, + envelope: RbcDagShadowCarrier, + ) -> Result<(), ShadowServiceErrorV1> { + validate_wire_size( + "carrier", + envelope.canonical_carrier.len(), + MAX_CARRIER_CONTENT_SIZE_V1, + )?; + validate_wire_size( + "authentication sidecar", + envelope.authentication_sidecar.len(), + self.max_sidecar_size, + )?; + self.send(ShadowServiceMessageV1::Carrier { peer, envelope }) + } + + pub(crate) fn carrier_request( + &self, + peer: AuthorityIndex, + reference: BlockReference, + ) -> Result<(), ShadowServiceErrorV1> { + self.send(ShadowServiceMessageV1::CarrierRequest { peer, reference }) + } + + pub(crate) fn carrier_response( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierResponse, + ) -> Result<(), ShadowServiceErrorV1> { + validate_wire_size( + "carrier response", + response.canonical_carrier.len(), + MAX_CARRIER_CONTENT_SIZE_V1, + )?; + self.send(ShadowServiceMessageV1::CarrierResponse { peer, response }) + } + + pub(crate) fn direct_delivered( + &self, + identity: ShadowDeliveryIdentityV1, + ) -> Result<(), ShadowServiceErrorV1> { + if identity.author as usize >= self.committee_size { + return Err(ShadowServiceErrorV1::UnknownAuthority(identity.author)); + } + if !self.desired_direct_deliveries.lock().insert(identity) { + return Ok(()); + } + match self + .sender + .try_send(ShadowServiceMessageV1::DirectDeliveriesChanged) + { + Ok(()) | Err(TrySendError::Full(_)) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), + } + } + + pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { + self.update_peer(peer, true) + } + + pub(crate) fn peer_disconnected( + &self, + peer: AuthorityIndex, + ) -> Result<(), ShadowServiceErrorV1> { + self.update_peer(peer, false) + } + + pub(crate) async fn shutdown(&self) -> Result<(), ShadowServiceErrorV1> { + let (reply, receiver) = oneshot::channel(); + self.sender + .send(ShadowServiceMessageV1::Shutdown(reply)) + .await + .map_err(|_| ShadowServiceErrorV1::Stopped)?; + receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped)? + } + + fn update_peer( + &self, + peer: AuthorityIndex, + connected: bool, + ) -> Result<(), ShadowServiceErrorV1> { + if peer as usize >= self.committee_size { + return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); + } + if peer == self.own_authority { + return Err(ShadowServiceErrorV1::Loopback(peer)); + } + let mut topology = self.desired_topology.lock(); + let state = topology.entry(peer).or_insert((false, 0)); + if state.0 != connected { + state.0 = connected; + state.1 = state.1.saturating_add(1); + } + drop(topology); + match self + .sender + .try_send(ShadowServiceMessageV1::TopologyChanged) + { + Ok(()) | Err(TrySendError::Full(_)) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), + } + } +} + +impl ShadowServiceMessageV1 { + fn kind(&self) -> &'static str { + match self { + Self::LocalCarrier(_) => "local", + Self::Carrier { .. } => "carrier", + Self::CarrierRequest { .. } => "carrier_request", + Self::CarrierResponse { .. } => "carrier_response", + Self::DirectDeliveriesChanged => "direct_deliveries_changed", + Self::TopologyChanged => "topology_changed", + Self::RetryRecovery => "recovery_retry", + Self::Shutdown(_) => "shutdown", + } + } +} + +#[derive(Debug)] +pub(crate) enum ShadowServiceEventV1 { + Ready, + ComparisonBacklog { + unpaired_direct: usize, + unpaired_shadow: usize, + max_round_lag: RoundNumber, + }, + Network { + recipient: AuthorityIndex, + message: NetworkMessage, + }, + Delivered(ShadowDeliveryIdentityV1), + Comparison(ShadowDeliveryComparisonV1), + Input { + kind: &'static str, + outcome: &'static str, + }, + WalDurable { + batches: u64, + records: u64, + }, + Recovered { + batches: u64, + discarded_tail_bytes: u64, + }, + PendingRecovery(usize), + Rejected { + peer: Option, + error: String, + }, +} + +#[derive(Debug)] +pub(crate) enum ShadowServiceErrorV1 { + Shadow(ShadowErrorV1), + StartTask(tokio::task::JoinError), + Stopped, + Overloaded { + kind: &'static str, + capacity: usize, + }, + BenchmarkInvalid { + reason: &'static str, + }, + InputTooLarge { + field: &'static str, + actual: usize, + maximum: usize, + }, + CommitteeBurstTooLarge { + committee_size: usize, + maximum_capacity: usize, + }, + UnknownAuthority(AuthorityIndex), + Loopback(AuthorityIndex), + ConflictingLocalHeader(RoundNumber), + MissingRecoveredLocalHeader(RoundNumber), + RecoveredLocalHeaderMismatch(RoundNumber), + LocalHeaderAuthority { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + UnauthenticatedCarrierRetained, + UnexpectedResponse(BlockReference), + ResponseFromNonHolder { + peer: AuthorityIndex, + reference: BlockReference, + }, +} + +impl fmt::Display for ShadowServiceErrorV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Shadow(error) => error.fmt(formatter), + Self::StartTask(error) => write!( + formatter, + "Starfish-RBC-DAG shadow startup task failed: {error}" + ), + Self::Stopped => formatter.write_str("Starfish-RBC-DAG shadow service stopped"), + Self::Overloaded { kind, capacity } => write!( + formatter, + "Starfish-RBC-DAG shadow {kind} input was dropped because the queue is full \ + (capacity {capacity}); benchmark comparison is invalid" + ), + Self::BenchmarkInvalid { reason } => write!( + formatter, + "Starfish-RBC-DAG shadow benchmark was disabled after dropping {reason} input" + ), + Self::InputTooLarge { + field, + actual, + maximum, + } => write!( + formatter, + "Starfish-RBC-DAG shadow {field} is {actual} bytes, above the {maximum}-byte limit" + ), + Self::CommitteeBurstTooLarge { + committee_size, + maximum_capacity, + } => write!( + formatter, + "Starfish-RBC-DAG shadow committee size {committee_size} needs a burst queue of \ + {}, above the memory-safe capacity limit {maximum_capacity}", + committee_size + .saturating_sub(1) + .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1), + ), + Self::UnknownAuthority(authority) => { + write!(formatter, "unknown shadow peer authority {authority}") + } + Self::Loopback(authority) => { + write!(formatter, "shadow peer authority {authority} is local") + } + Self::ConflictingLocalHeader(round) => write!( + formatter, + "conflicting direct headers supplied for queued shadow round {round}" + ), + Self::MissingRecoveredLocalHeader(round) => write!( + formatter, + "persisted shadow carrier at round {round} has no matching recovered direct header" + ), + Self::RecoveredLocalHeaderMismatch(round) => write!( + formatter, + "persisted shadow carrier and recovered direct header disagree at round {round}" + ), + Self::LocalHeaderAuthority { expected, actual } => write!( + formatter, + "shadow local header authority {actual} does not match local authority {expected}" + ), + Self::UnauthenticatedCarrierRetained => formatter.write_str( + "shadow carrier authentication failed; canonical content was retained candidate-only", + ), + Self::UnexpectedResponse(reference) => { + write!(formatter, "unexpected shadow response for {reference}") + } + Self::ResponseFromNonHolder { peer, reference } => write!( + formatter, + "shadow response for {reference} came from non-holder {peer}" + ), + } + } +} + +impl Error for ShadowServiceErrorV1 { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Shadow(error) => Some(error), + Self::StartTask(error) => Some(error), + _ => None, + } + } +} + +impl From for ShadowServiceErrorV1 { + fn from(error: ShadowErrorV1) -> Self { + Self::Shadow(error) + } +} + +pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, +> { + let committee_size = committee.committee().len(); + let input_capacity = shadow_input_capacity(committee_size)?; + let max_sidecar_size = + authentication_sidecar_size(context.authentication_scheme(), committee_size); + let path = path.as_ref().to_path_buf(); + let mut pending_local = BTreeMap::new(); + for header in recovered_local_headers { + let local = ShadowLocalCarrierV1::from_direct_header(&header); + if local.author != own_authority { + return Err(ShadowServiceErrorV1::LocalHeaderAuthority { + expected: own_authority, + actual: local.author, + }); + } + if let Some(previous) = pending_local.insert(local.round, local) { + if previous != local { + return Err(ShadowServiceErrorV1::ConflictingLocalHeader(local.round)); + } + } + } + let (message_tx, message_rx) = mpsc::channel(input_capacity); + let (event_tx, event_rx) = mpsc::channel(SHADOW_SERVICE_EVENT_CAPACITY_V1); + let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); + let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); + let invalidated_by_overload = Arc::new(Mutex::new(None)); + let retry_tx = message_tx.downgrade(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(SHADOW_RECOVERY_RETRY_INTERVAL_V1); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + interval.tick().await; + let Some(retry_tx) = retry_tx.upgrade() else { + break; + }; + match retry_tx.try_send(ShadowServiceMessageV1::RetryRecovery) { + Ok(()) | Err(TrySendError::Full(_)) => {} + Err(TrySendError::Closed(_)) => break, + } + } + }); + let startup_events = event_tx.clone(); + let actor_desired_topology = Arc::clone(&desired_topology); + let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); + let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); + let task = tokio::spawn(async move { + let opened = tokio::task::spawn_blocking(move || { + StarfishRbcDagShadowV1::open(path, committee, own_authority, context, authorizer) + }) + .await; + let (core, open_report) = match opened { + Ok(Ok(opened)) => opened, + Ok(Err(error)) => { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: error.to_string(), + }) + .await; + return; + } + Err(error) => { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::StartTask(error).to_string(), + }) + .await; + return; + } + }; + let persisted_local = match core.local_outbound_metadata() { + Ok(metadata) => metadata + .into_iter() + .map(|(round, commitment, creation_time_ns)| { + (round, (commitment, creation_time_ns)) + }) + .collect::>(), + Err(error) => { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: error.to_string(), + }) + .await; + return; + } + }; + let durable_round = core.local_carrier_round(); + for (round, (commitment, creation_time_ns)) in &persisted_local { + let Some(recovered) = pending_local.get(round) else { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::MissingRecoveredLocalHeader(*round) + .to_string(), + }) + .await; + return; + }; + if recovered.transactions_commitment != *commitment + || recovered.creation_time_ns != *creation_time_ns + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) + .to_string(), + }) + .await; + return; + } + } + if let Some(round) = pending_local + .keys() + .copied() + .find(|round| *round < durable_round && !persisted_local.contains_key(round)) + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(round).to_string(), + }) + .await; + return; + } + let reported_shadow_deliveries = match core.delivered_identities() { + Ok(identities) => identities.into_iter().collect::>(), + Err(error) => { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: error.to_string(), + }) + .await; + return; + } + }; + let recovered_shadow_deliveries = reported_shadow_deliveries.clone(); + let reported_shadow_delivery_slots = reported_shadow_deliveries + .iter() + .map(delivery_slot) + .collect(); + let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); + pending_local.retain(|round, _| *round >= core.local_carrier_round()); + let state = ShadowServiceStateV1 { + core, + own_authority, + committee_size, + events: event_tx, + connected: BTreeSet::new(), + desired_topology: actor_desired_topology, + desired_direct_deliveries: actor_desired_direct_deliveries, + observed_topology: BTreeMap::new(), + invalidated_by_overload: actor_invalidated_by_overload, + pending_local, + pending_recovery: BTreeMap::new(), + recovery_last_attempt: BTreeMap::new(), + direct_deliveries: BTreeSet::new(), + reported_shadow_deliveries, + recovered_shadow_deliveries, + comparison_backlog, + reported_matches: BTreeSet::new(), + reported_mismatches: BTreeSet::new(), + reported_conflicts: BTreeSet::new(), + fatal: false, + }; + if let Err(error) = tokio::task::spawn_blocking(move || { + run_shadow_service(state, message_rx, open_report); + }) + .await + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: format!("Starfish-RBC-DAG shadow actor task failed: {error}"), + }) + .await; + } + }); + Ok(( + StarfishRbcDagShadowServiceHandleV1 { + sender: message_tx, + max_sidecar_size, + own_authority, + committee_size, + input_capacity, + desired_topology, + desired_direct_deliveries, + invalidated_by_overload, + }, + event_rx, + task, + )) +} + +struct ShadowComparisonBacklogV1 { + direct_slots: BTreeSet, + reported_shadow_slots: BTreeSet, + // Recovered shadow slots are deliberately absent from this set: they may + // pair a current direct observation but are not current-process timing + // observations and therefore cannot create shadow-only backlog. + epoch_shadow_slots: BTreeSet, + unpaired_direct_slots: BTreeSet, + unpaired_shadow_slots: BTreeSet, + latest_epoch_round: RoundNumber, +} + +impl ShadowComparisonBacklogV1 { + fn new(reported_shadow_slots: BTreeSet) -> Self { + Self { + direct_slots: BTreeSet::new(), + reported_shadow_slots, + epoch_shadow_slots: BTreeSet::new(), + unpaired_direct_slots: BTreeSet::new(), + unpaired_shadow_slots: BTreeSet::new(), + latest_epoch_round: 0, + } + } + + fn observe_direct(&mut self, slot: ShadowDeliverySlotV1) { + self.observe_epoch_round(slot); + if self.direct_slots.insert(slot) { + if !self.reported_shadow_slots.contains(&slot) { + self.unpaired_direct_slots.insert(slot); + } + self.unpaired_shadow_slots.remove(&slot); + } + } + + fn observe_epoch_shadow(&mut self, slot: ShadowDeliverySlotV1) { + self.observe_epoch_round(slot); + self.reported_shadow_slots.insert(slot); + if self.epoch_shadow_slots.insert(slot) { + if !self.direct_slots.contains(&slot) { + self.unpaired_shadow_slots.insert(slot); + } + self.unpaired_direct_slots.remove(&slot); + } + } + + fn observe_epoch_round(&mut self, slot: ShadowDeliverySlotV1) { + self.latest_epoch_round = self.latest_epoch_round.max(slot.round); + } + + fn counts(&self) -> (usize, usize, RoundNumber) { + let max_round_lag = self + .unpaired_direct_slots + .union(&self.unpaired_shadow_slots) + .map(|slot| self.latest_epoch_round.saturating_sub(slot.round)) + .max() + .unwrap_or(0); + ( + self.unpaired_direct_slots.len(), + self.unpaired_shadow_slots.len(), + max_round_lag, + ) + } +} + +struct ShadowServiceStateV1 { + core: StarfishRbcDagShadowV1, + own_authority: AuthorityIndex, + committee_size: usize, + events: mpsc::Sender, + connected: BTreeSet, + desired_topology: Arc>>, + desired_direct_deliveries: Arc>>, + observed_topology: BTreeMap, + invalidated_by_overload: Arc>>, + pending_local: BTreeMap, + pending_recovery: BTreeMap>, + recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, + direct_deliveries: BTreeSet, + reported_shadow_deliveries: BTreeSet, + recovered_shadow_deliveries: BTreeSet, + comparison_backlog: ShadowComparisonBacklogV1, + reported_matches: BTreeSet, + reported_mismatches: BTreeSet<(ShadowDeliveryIdentityV1, ShadowDeliveryIdentityV1)>, + reported_conflicts: BTreeSet, + fatal: bool, +} + +impl ShadowServiceStateV1 { + fn emit(&self, event: ShadowServiceEventV1) { + let _ = self.events.blocking_send(event); + } + + fn reject(&self, peer: Option, error: impl fmt::Display) { + self.emit(ShadowServiceEventV1::Rejected { + peer, + error: error.to_string(), + }); + } + + fn emit_comparison_backlog(&self) { + let (unpaired_direct, unpaired_shadow, max_round_lag) = self.comparison_backlog.counts(); + self.emit(ShadowServiceEventV1::ComparisonBacklog { + unpaired_direct, + unpaired_shadow, + max_round_lag, + }); + } + + fn validate_peer(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { + if peer as usize >= self.committee_size { + return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); + } + if peer == self.own_authority { + return Err(ShadowServiceErrorV1::Loopback(peer)); + } + Ok(()) + } + + fn reconcile_topology(&mut self) { + let desired = self.desired_topology.lock().clone(); + let mut newly_connected = Vec::new(); + for (peer, state) in &desired { + if self.observed_topology.get(peer) == Some(state) { + continue; + } + self.recovery_last_attempt + .retain(|(_, holder), _| holder != peer); + if state.0 { + self.connected.insert(*peer); + newly_connected.push(*peer); + } else { + self.connected.remove(peer); + } + } + self.observed_topology = desired; + if !newly_connected.is_empty() { + let retransmissions = self.core.retransmissions(); + for peer in newly_connected { + for envelope in &retransmissions { + self.send_envelope(peer, envelope); + } + } + self.flush_recovery_requests(); + } + } + + fn reconcile_direct_deliveries(&mut self) { + let desired = self.desired_direct_deliveries.lock().clone(); + let newly_observed = desired + .difference(&self.direct_deliveries) + .copied() + .collect::>(); + for identity in newly_observed { + self.direct_deliveries.insert(identity); + let slot = delivery_slot(&identity); + self.comparison_backlog.observe_direct(slot); + self.emit(ShadowServiceEventV1::Input { + kind: "delivery", + outcome: "direct", + }); + self.emit_slot_comparison(slot); + } + if !desired.is_empty() { + self.emit_comparison_backlog(); + } + } + + fn observe_external_invalidation(&mut self) -> bool { + let reason = *self.invalidated_by_overload.lock(); + if let Some(reason) = reason { + self.mark_fatal(ShadowServiceErrorV1::BenchmarkInvalid { reason }); + true + } else { + false + } + } + + fn mark_fatal(&mut self, error: impl fmt::Display) { + if self.fatal { + return; + } + self.fatal = true; + if self.invalidated_by_overload.lock().is_none() { + *self.invalidated_by_overload.lock() = Some("fatal_core"); + } + self.emit(ShadowServiceEventV1::Input { + kind: "benchmark", + outcome: "invalid", + }); + self.reject(None, error); + } + + fn broadcast(&self, envelope: &ShadowOutboundEnvelopeV1) { + for recipient in 0..self.committee_size { + let recipient = recipient as AuthorityIndex; + if recipient != self.own_authority { + self.send_envelope(recipient, envelope); + } + } + } + + fn send_envelope(&self, recipient: AuthorityIndex, envelope: &ShadowOutboundEnvelopeV1) { + self.emit(ShadowServiceEventV1::Network { + recipient, + message: NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { + canonical_carrier: envelope.canonical_carrier_wire().to_vec(), + authentication_sidecar: envelope.authentication_sidecar().to_vec(), + }), + }); + } + + fn report_wal_delta(&self, before: (u64, u64)) { + let after = self.core.wal_counts(); + let batches = after.0.saturating_sub(before.0); + let records = after.1.saturating_sub(before.1); + if batches != 0 || records != 0 { + self.emit(ShadowServiceEventV1::WalDurable { batches, records }); + } + } + + fn process_effects(&mut self, effects: Vec) { + for effect in effects { + match effect { + ModelEffect::NeedCarrier { target, holders } => { + self.pending_recovery.entry(target).or_default().extend( + holders.into_iter().filter(|holder| { + *holder != self.own_authority + && (*holder as usize) < self.committee_size + }), + ); + } + ModelEffect::Delivered(reference) => { + self.pending_recovery.remove(&reference); + } + ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} + } + } + self.reconcile_pending_recovery(); + self.flush_recovery_requests(); + self.emit(ShadowServiceEventV1::PendingRecovery( + self.pending_recovery.len(), + )); + self.report_new_shadow_deliveries(); + } + + fn reconcile_pending_recovery(&mut self) { + self.pending_recovery + .retain(|target, _| self.core.retained_candidate_wire(*target).is_none()); + self.recovery_last_attempt.retain(|(target, holder), _| { + self.pending_recovery + .get(target) + .is_some_and(|holders| holders.contains(holder)) + }); + } + + fn enqueue_local(&mut self, local: ShadowLocalCarrierV1) { + if local.author != self.own_authority { + self.reject( + None, + ShadowServiceErrorV1::LocalHeaderAuthority { + expected: self.own_authority, + actual: local.author, + }, + ); + return; + } + let durable_round = self.core.local_carrier_round(); + if local.round < durable_round { + self.emit(ShadowServiceEventV1::Input { + kind: "local", + outcome: "already_durable", + }); + return; + } + if let Some(existing) = self.pending_local.get(&local.round) { + if existing == &local { + self.emit(ShadowServiceEventV1::Input { + kind: "local", + outcome: "duplicate", + }); + } else { + self.reject( + None, + ShadowServiceErrorV1::ConflictingLocalHeader(local.round), + ); + } + return; + } + let round = local.round; + self.pending_local.insert(round, local); + if round != durable_round || !self.core.can_create_carrier() { + self.emit(ShadowServiceEventV1::Input { + kind: "local", + outcome: "queued", + }); + } + self.retry_pending_local(); + } + + /// Create only the exact round opened by the durable carrier clock. A + /// future direct header remains queued until authenticated carrier input + /// advances the model; recovered historical headers below that clock are + /// harmless idempotent replays. + fn retry_pending_local(&mut self) { + loop { + let durable_round = self.core.local_carrier_round(); + self.pending_local + .retain(|round, _| *round >= durable_round); + if !self.core.can_create_carrier() { + return; + } + let Some(local) = self.pending_local.remove(&durable_round) else { + return; + }; + let before = self.core.wal_counts(); + match self.core.create_local_carrier( + local.round, + local.transactions_commitment, + local.creation_time_ns, + ) { + Ok((envelope, effects)) => { + self.emit(ShadowServiceEventV1::Input { + kind: "local", + outcome: "accepted", + }); + self.report_wal_delta(before); + self.broadcast(&envelope); + self.process_effects(effects); + } + Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { + self.pending_local.insert(local.round, local); + self.emit(ShadowServiceEventV1::Input { + kind: "local", + outcome: "queued", + }); + return; + } + Err(error) => { + self.emit(ShadowServiceEventV1::Input { + kind: "local", + outcome: "rejected", + }); + self.mark_fatal(error); + return; + } + } + } + } + + fn flush_recovery_requests(&mut self) { + let now = Instant::now(); + let mut requests = Vec::new(); + for (reference, holders) in &self.pending_recovery { + for holder in holders { + if self.connected.contains(holder) + && self + .recovery_last_attempt + .get(&(*reference, *holder)) + .is_none_or(|last| { + now.saturating_duration_since(*last) + >= SHADOW_RECOVERY_RETRY_INTERVAL_V1 + }) + { + requests.push((*reference, *holder)); + } + } + } + for (reference, holder) in requests { + self.recovery_last_attempt.insert((reference, holder), now); + self.emit(ShadowServiceEventV1::Network { + recipient: holder, + message: NetworkMessage::RbcDagShadowCarrierRequest(reference), + }); + } + } + + fn report_new_shadow_deliveries(&mut self) { + let identities: BTreeSet<_> = match self.core.delivered_identities() { + Ok(identities) => identities.into_iter().collect(), + Err(error) => { + self.reject(None, error); + return; + } + }; + let new_identities = identities + .difference(&self.reported_shadow_deliveries) + .copied() + .collect::>(); + self.reported_shadow_deliveries = identities; + for identity in &new_identities { + let slot = delivery_slot(identity); + self.comparison_backlog.observe_epoch_shadow(slot); + self.emit(ShadowServiceEventV1::Delivered(*identity)); + self.emit_slot_comparison(slot); + self.emit_comparison_backlog(); + } + } + + fn emit_slot_comparison(&mut self, slot: ShadowDeliverySlotV1) { + let direct = self + .direct_deliveries + .iter() + .filter(|identity| delivery_slot(identity) == slot) + .copied() + .collect::>(); + let shadow = self + .reported_shadow_deliveries + .iter() + .filter(|identity| delivery_slot(identity) == slot) + .copied() + .collect::>(); + if direct.is_empty() || shadow.is_empty() { + return; + } + if direct.len() > 1 || shadow.len() > 1 { + if self.reported_conflicts.insert(slot) { + self.emit(ShadowServiceEventV1::Comparison( + ShadowDeliveryComparisonV1::Ambiguous { slots: vec![slot] }, + )); + } + return; + } + let direct = direct[0]; + let shadow = shadow[0]; + if direct == shadow { + if self.reported_matches.insert(direct) { + if self.recovered_shadow_deliveries.contains(&shadow) { + self.emit(ShadowServiceEventV1::Input { + kind: "comparison", + outcome: "recovered_match", + }); + } + self.emit(ShadowServiceEventV1::Comparison( + ShadowDeliveryComparisonV1::Match, + )); + } + } else if self.reported_mismatches.insert((direct, shadow)) { + self.emit(ShadowServiceEventV1::Comparison( + ShadowDeliveryComparisonV1::Mismatch { + direct_only: vec![direct], + shadow_only: vec![shadow], + }, + )); + } + } +} + +fn run_shadow_service( + mut state: ShadowServiceStateV1, + mut messages: mpsc::Receiver, + open_report: ShadowOpenReportV1, +) { + if open_report.replayed_batches() != 0 || open_report.discarded_tail_bytes() != 0 { + state.emit(ShadowServiceEventV1::Recovered { + batches: open_report.replayed_batches(), + discarded_tail_bytes: open_report.discarded_tail_bytes(), + }); + } + state.reconcile_topology(); + state.reconcile_direct_deliveries(); + if !state.observe_external_invalidation() { + state.emit(ShadowServiceEventV1::Ready); + state.emit_comparison_backlog(); + state.process_effects(open_report.recovery_effects().to_vec()); + state.retry_pending_local(); + } + + while !state.fatal { + let Some(message) = messages.blocking_recv() else { + break; + }; + let message = match message { + ShadowServiceMessageV1::Shutdown(reply) => { + let events = state.events.clone(); + let result = state + .core + .shutdown() + .map(|_| ()) + .map_err(ShadowServiceErrorV1::from); + if reply.send(result).is_err() { + let _ = events.blocking_send(ShadowServiceEventV1::Rejected { + peer: None, + error: "shadow shutdown acknowledgment receiver was dropped".to_owned(), + }); + } + return; + } + message => message, + }; + if state.observe_external_invalidation() { + break; + } + match message { + ShadowServiceMessageV1::LocalCarrier(local) => { + state.enqueue_local(local); + } + ShadowServiceMessageV1::Carrier { peer, envelope } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + let before = state.core.wal_counts(); + match state.core.receive_or_retain_from_peer( + &envelope.canonical_carrier, + &envelope.authentication_sidecar, + peer, + ) { + Ok(outcome) => { + let outcome_label = match outcome.disposition() { + ShadowIngressDispositionV1::Authenticated => "authenticated", + ShadowIngressDispositionV1::CandidateRetained => { + "retained_unauthenticated" + } + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { + "ignored" + } + }; + state.emit(ShadowServiceEventV1::Input { + kind: "carrier", + outcome: outcome_label, + }); + state.report_wal_delta(before); + state.process_effects(outcome.effects().to_vec()); + state.retry_pending_local(); + if outcome.disposition() == ShadowIngressDispositionV1::CandidateRetained { + state.reject( + Some(peer), + ShadowServiceErrorV1::UnauthenticatedCarrierRetained, + ); + } + } + Err(error) => { + state.emit(ShadowServiceEventV1::Input { + kind: "carrier", + outcome: "rejected", + }); + if is_fatal_core_error(&error) { + state.mark_fatal(error); + } else { + state.reject(Some(peer), error); + } + } + } + } + ShadowServiceMessageV1::CarrierRequest { peer, reference } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + if let Some(canonical_carrier) = state.core.retained_candidate_wire(reference) { + state.emit(ShadowServiceEventV1::Network { + recipient: peer, + message: NetworkMessage::RbcDagShadowCarrierResponse( + RbcDagShadowCarrierResponse { + reference, + canonical_carrier, + }, + ), + }); + } + } + ShadowServiceMessageV1::CarrierResponse { peer, response } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + let Some(holders) = state.pending_recovery.get(&response.reference) else { + state.reject( + Some(peer), + ShadowServiceErrorV1::UnexpectedResponse(response.reference), + ); + continue; + }; + if !holders.contains(&peer) { + state.reject( + Some(peer), + ShadowServiceErrorV1::ResponseFromNonHolder { + peer, + reference: response.reference, + }, + ); + continue; + } + let before = state.core.wal_counts(); + match state + .core + .recover_candidate_for(response.reference, &response.canonical_carrier) + { + Ok(effects) => { + state.pending_recovery.remove(&response.reference); + state + .recovery_last_attempt + .retain(|(target, _), _| *target != response.reference); + state.emit(ShadowServiceEventV1::Input { + kind: "recovery", + outcome: "accepted", + }); + state.report_wal_delta(before); + state.process_effects(effects); + state.retry_pending_local(); + } + Err(error) => { + state.emit(ShadowServiceEventV1::Input { + kind: "recovery", + outcome: "rejected", + }); + if is_fatal_core_error(&error) { + state.mark_fatal(error); + } else { + state.reject(Some(peer), error); + } + } + } + } + ShadowServiceMessageV1::DirectDeliveriesChanged => { + state.reconcile_direct_deliveries(); + } + ShadowServiceMessageV1::TopologyChanged => state.reconcile_topology(), + ShadowServiceMessageV1::RetryRecovery => { + state.reconcile_topology(); + state.reconcile_pending_recovery(); + state.flush_recovery_requests(); + } + ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), + } + state.reconcile_topology(); + state.reconcile_direct_deliveries(); + } + let events = state.events.clone(); + if let Err(error) = state.core.shutdown() { + let _ = events.blocking_send(ShadowServiceEventV1::Rejected { + peer: None, + error: error.to_string(), + }); + } +} + +fn delivery_slot(identity: &ShadowDeliveryIdentityV1) -> ShadowDeliverySlotV1 { + ShadowDeliverySlotV1 { + author: identity.author, + round: identity.round, + } +} + +fn authentication_sidecar_size(scheme: BlockAuthenticationScheme, committee_size: usize) -> usize { + const SIDECAR_HEADER_SIZE: usize = 3; + SIDECAR_HEADER_SIZE + + match scheme { + BlockAuthenticationScheme::Ed25519 => SIGNATURE_SIZE, + BlockAuthenticationScheme::MlDsa44 => ML_DSA_44_SIGNATURE_SIZE, + BlockAuthenticationScheme::MlDsa65 => ML_DSA_65_SIGNATURE_SIZE, + BlockAuthenticationScheme::MacVector => committee_size.saturating_mul(MAC_TAG_SIZE), + } +} + +fn shadow_input_capacity(committee_size: usize) -> Result { + let committee_burst = committee_size + .saturating_sub(1) + .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1); + if committee_burst > SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1 { + return Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { + committee_size, + maximum_capacity: SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1, + }); + } + Ok(committee_burst.max(SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1)) +} + +fn validate_wire_size( + field: &'static str, + actual: usize, + maximum: usize, +) -> Result<(), ShadowServiceErrorV1> { + if actual > maximum { + Err(ShadowServiceErrorV1::InputTooLarge { + field, + actual, + maximum, + }) + } else { + Ok(()) + } +} + +fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { + matches!( + error, + ShadowErrorV1::Wal(_) + | ShadowErrorV1::PostDurabilityCommit(_) + | ShadowErrorV1::PostDurabilityJournal(_) + | ShadowErrorV1::Poisoned + ) +} + +#[cfg(test)] +mod tests { + use std::{sync::Arc, time::Duration}; + + use tempfile::TempDir; + use tokio::time::timeout; + + use super::*; + use crate::{ + committee::Committee, + crypto::{TransactionsCommitment, mac_keyrings_for_test}, + starfish_rbc_dag::{ + CandidateCarrierV1, CarrierAuthorizerV1, CarrierHeaderV1Args, RbcDagProtocolInstanceId, + RbcPhaseStatementV1, carrier_genesis_reference, + }, + types::{BlockDigest, VerifiedBlock}, + }; + + const N: usize = 4; + const EVENT_TIMEOUT: Duration = Duration::from_secs(5); + + struct Harness { + _directory: TempDir, + committee: RbcDagCommitteeContextV1, + context: RbcDagContextV1, + keyrings: Vec>, + paths: Vec, + } + + impl Harness { + fn new() -> Self { + let committee = Committee::new_test(vec![1; N]); + let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ); + let directory = tempfile::tempdir().unwrap(); + let paths = (0..N) + .map(|authority| directory.path().join(format!("shadow-{authority}.wal"))) + .collect(); + Self { + _directory: directory, + committee, + context, + keyrings: mac_keyrings_for_test(N), + paths, + } + } + + fn start( + &self, + authority: AuthorityIndex, + recovered: Vec, + ) -> ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ) { + start_starfish_rbc_dag_shadow_service_v1( + &self.paths[authority as usize], + self.committee.clone(), + authority, + self.context, + ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), + recovered, + ) + .unwrap() + } + + fn envelope( + &self, + candidate: &CandidateCarrierV1, + author: AuthorityIndex, + ) -> RbcDagShadowCarrier { + let authentication = self + .context + .authenticate_with_committee( + candidate, + &self.committee, + CarrierAuthorizerV1::MacVector { + authority: author, + keys: &self.keyrings[author as usize], + }, + ) + .unwrap(); + RbcDagShadowCarrier { + canonical_carrier: candidate.canonical_wire_bytes().unwrap(), + authentication_sidecar: authentication.canonical_wire_bytes(), + } + } + } + + async fn next_event(events: &mut mpsc::Receiver) -> ShadowServiceEventV1 { + timeout(EVENT_TIMEOUT, events.recv()) + .await + .expect("shadow actor timed out") + .expect("shadow actor stopped") + } + + async fn wait_ready(events: &mut mpsc::Receiver) { + loop { + match next_event(events).await { + ShadowServiceEventV1::Ready => return, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("shadow startup failed: {error}") + } + _ => {} + } + } + } + + async fn wait_backlog( + events: &mut mpsc::Receiver, + expected: (usize, usize, RoundNumber), + ) { + loop { + match next_event(events).await { + ShadowServiceEventV1::ComparisonBacklog { + unpaired_direct, + unpaired_shadow, + max_round_lag, + } if (unpaired_direct, unpaired_shadow, max_round_lag) == expected => return, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("shadow service rejected input while waiting for backlog: {error}") + } + _ => {} + } + } + } + + async fn next_carrier( + events: &mut mpsc::Receiver, + expected_recipient: AuthorityIndex, + ) -> RbcDagShadowCarrier { + loop { + if let ShadowServiceEventV1::Network { + recipient, + message: NetworkMessage::RbcDagShadowCarrier(envelope), + } = next_event(events).await + { + if recipient == expected_recipient { + return envelope; + } + } + } + } + + async fn stop( + handle: StarfishRbcDagShadowServiceHandleV1, + events: mpsc::Receiver, + task: JoinHandle<()>, + ) { + drop(events); + handle.shutdown().await.unwrap(); + task.await.unwrap(); + } + + async fn startup_rejection(mut events: mpsc::Receiver) -> String { + loop { + match next_event(&mut events).await { + ShadowServiceEventV1::Rejected { peer: None, error } => return error, + ShadowServiceEventV1::Ready => panic!("invalid shadow startup became ready"), + _ => {} + } + } + } + + fn direct_header(author: AuthorityIndex, round: RoundNumber, marker: u8) -> RbcCanonicalHeader { + RbcCanonicalHeader::try_new( + author, + round, + (0..N) + .map(|authority| { + *VerifiedBlock::new_genesis(authority as AuthorityIndex).reference() + }) + .collect(), + Vec::new(), + u64::from(round) * 1_000 + u64::from(marker), + TransactionsCommitment::from_bytes([marker; 32]), + ) + .unwrap() + } + + fn round_one_candidate( + author: AuthorityIndex, + committee: &RbcDagCommitteeContextV1, + marker: u8, + ) -> CandidateCarrierV1 { + let weak_parents = committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(carrier_genesis_reference) + .collect(); + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 1, + own_prev: carrier_genesis_reference(author), + weak_parents, + transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: u64::from(marker), + }, + committee, + ) + .unwrap() + } + + #[test] + fn comparison_backlog_excludes_recovery_and_ages_old_holes() { + let recovered = ShadowDeliverySlotV1 { + author: 0, + round: 10, + }; + let mut backlog = ShadowComparisonBacklogV1::new(BTreeSet::from([recovered])); + assert_eq!(backlog.counts(), (0, 0, 0)); + + // A direct replay can pair with recovered shadow state without + // inventing a current-epoch shadow observation. + backlog.observe_direct(recovered); + assert_eq!(backlog.counts(), (0, 0, 0)); + + let old_hole = ShadowDeliverySlotV1 { + author: 1, + round: 11, + }; + backlog.observe_direct(old_hole); + assert_eq!(backlog.counts(), (1, 0, 0)); + + let newer = ShadowDeliverySlotV1 { + author: 2, + round: 15, + }; + backlog.observe_direct(newer); + backlog.observe_epoch_shadow(newer); + assert_eq!(backlog.counts(), (1, 0, 4)); + + backlog.observe_epoch_shadow(old_hole); + assert_eq!(backlog.counts(), (0, 0, 0)); + } + + #[tokio::test] + async fn service_emits_initial_and_direct_backlog() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start(0, Vec::new()); + wait_ready(&mut events).await; + wait_backlog(&mut events, (0, 0, 0)).await; + + handle + .direct_delivered(ShadowDeliveryIdentityV1::new( + 1, + 7, + TransactionsCommitment::from_bytes([0xB7; 32]), + )) + .unwrap(); + wait_backlog(&mut events, (1, 0, 0)).await; + + stop(handle, events, task).await; + } + + fn phase_carrier( + author: AuthorityIndex, + statement: RbcPhaseStatementV1, + committee: &RbcDagCommitteeContextV1, + ) -> CandidateCarrierV1 { + let previous = |authority: AuthorityIndex| BlockReference { + authority, + round: 1, + digest: BlockDigest::from([0xA0 + authority as u8; 32]), + }; + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 2, + own_prev: previous(author), + weak_parents: committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(previous) + .collect(), + transactions_commitment: TransactionsCommitment::from_bytes( + [0xB0 + author as u8; 32], + ), + data_acknowledgments: Vec::new(), + phase_batch: vec![statement], + consensus_vertex: None, + creation_time_ns: 2, + }, + committee, + ) + .unwrap() + } + + #[tokio::test] + async fn restart_replays_matching_and_rejects_inconsistent_direct_history() { + let harness = Harness::new(); + let direct = direct_header(0, 1, 0x41); + let (handle, mut events, task) = harness.start(0, vec![direct.clone()]); + wait_ready(&mut events).await; + let original = next_carrier(&mut events, 1).await; + stop(handle, events, task).await; + + let (restarted, mut restarted_events, restarted_task) = + harness.start(0, vec![direct.clone()]); + let mut replayed = false; + loop { + match next_event(&mut restarted_events).await { + ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches > 0, + ShadowServiceEventV1::Ready => break, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("valid shadow restart failed: {error}") + } + _ => {} + } + } + assert!(replayed, "valid shadow restart did not replay its WAL"); + restarted.peer_connected(1).unwrap(); + assert_eq!(next_carrier(&mut restarted_events, 1).await, original); + stop(restarted, restarted_events, restarted_task).await; + + let (missing_handle, missing_events, missing_task) = harness.start(0, Vec::new()); + let missing = startup_rejection(missing_events).await; + assert!(missing.contains("has no matching recovered direct header")); + drop(missing_handle); + missing_task.await.unwrap(); + + let divergent = direct_header(0, 1, 0x42); + let (divergent_handle, divergent_events, divergent_task) = + harness.start(0, vec![divergent]); + let mismatch = startup_rejection(divergent_events).await; + assert!(mismatch.contains("disagree at round 1")); + drop(divergent_handle); + divergent_task.await.unwrap(); + } + + #[tokio::test] + async fn local_carrier_broadcasts_one_identical_full_mac_vector_per_peer() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start(0, Vec::new()); + wait_ready(&mut events).await; + handle.local_header(&direct_header(0, 1, 0x11)).unwrap(); + + let mut by_recipient = BTreeMap::new(); + while by_recipient.len() < N - 1 { + if let ShadowServiceEventV1::Network { + recipient, + message: NetworkMessage::RbcDagShadowCarrier(envelope), + } = next_event(&mut events).await + { + by_recipient.insert(recipient, envelope); + } + } + assert_eq!( + by_recipient.keys().copied().collect::>(), + vec![1, 2, 3] + ); + let first = &by_recipient[&1]; + assert_eq!(first.authentication_sidecar.len(), 3 + N * MAC_TAG_SIZE); + for envelope in by_recipient.values() { + assert_eq!(envelope, first); + } + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &first.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + assert_eq!(candidate.header().carrier_round(), 1); + stop(handle, events, task).await; + } + + #[tokio::test] + async fn poisoned_receiver_mac_is_retained_candidate_only_and_served() { + let harness = Harness::new(); + let (author, mut author_events, author_task) = harness.start(0, Vec::new()); + wait_ready(&mut author_events).await; + author.local_header(&direct_header(0, 1, 0x21)).unwrap(); + let mut envelope = next_carrier(&mut author_events, 2).await; + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &envelope.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + envelope.authentication_sidecar[3 + 2 * MAC_TAG_SIZE] ^= 1; + + let (receiver, mut receiver_events, receiver_task) = harness.start(2, Vec::new()); + wait_ready(&mut receiver_events).await; + receiver.carrier(0, envelope.clone()).unwrap(); + let mut retained = false; + let mut rejected = false; + while !retained || !rejected { + match next_event(&mut receiver_events).await { + ShadowServiceEventV1::Input { + kind: "carrier", + outcome: "retained_unauthenticated", + } => retained = true, + ShadowServiceEventV1::Rejected { peer: Some(0), .. } => rejected = true, + ShadowServiceEventV1::Delivered(_) => { + panic!("candidate-only retention must not grant admission/delivery") + } + _ => {} + } + } + receiver.carrier_request(1, candidate.reference()).unwrap(); + loop { + if let ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrierResponse(response), + } = next_event(&mut receiver_events).await + { + assert_eq!(response.reference, candidate.reference()); + assert_eq!(response.canonical_carrier, envelope.canonical_carrier); + break; + } + } + stop(author, author_events, author_task).await; + stop(receiver, receiver_events, receiver_task).await; + } + + #[tokio::test] + async fn reconnect_replays_exact_persisted_envelope() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start(0, Vec::new()); + wait_ready(&mut events).await; + handle.local_header(&direct_header(0, 1, 0x31)).unwrap(); + let original = next_carrier(&mut events, 1).await; + handle.peer_connected(1).unwrap(); + let first_replay = next_carrier(&mut events, 1).await; + assert_eq!(first_replay, original); + handle.peer_disconnected(1).unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + handle.peer_connected(1).unwrap(); + let second_replay = next_carrier(&mut events, 1).await; + assert_eq!(second_replay, original); + stop(handle, events, task).await; + } + + #[tokio::test] + async fn queued_round_waits_for_shadow_quorum_then_broadcasts_exact_next_round() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start(0, Vec::new()); + wait_ready(&mut events).await; + let round_two = direct_header(0, 2, 0x42); + handle.local_header(&round_two).unwrap(); + handle.local_header(&direct_header(0, 1, 0x41)).unwrap(); + let round_one_wire = next_carrier(&mut events, 1).await; + let round_one = CandidateCarrierV1::decode_wire_with_committee( + &round_one_wire.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + assert_eq!(round_one.header().carrier_round(), 1); + + for author in [1, 2] { + let candidate = round_one_candidate(author, &harness.committee, 0x50 + author as u8); + handle + .carrier(author, harness.envelope(&candidate, author)) + .unwrap(); + } + let round_two_wire = loop { + let envelope = next_carrier(&mut events, 1).await; + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &envelope.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + if candidate.header().carrier_round() == 2 { + break candidate; + } + }; + assert_eq!( + round_two_wire.header().transactions_commitment(), + round_two.transactions_commitment() + ); + stop(handle, events, task).await; + } + + #[tokio::test] + async fn recovery_binds_holder_and_reference_then_compares_only_paired_slot_once() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start(3, Vec::new()); + wait_ready(&mut events).await; + handle.peer_connected(0).unwrap(); + handle.peer_connected(1).unwrap(); + let target = round_one_candidate(2, &harness.committee, 0x61); + for sender in [0, 1] { + let outer = phase_carrier( + sender, + RbcPhaseStatementV1::Ready { + target: target.reference(), + }, + &harness.committee, + ); + handle + .carrier(sender, harness.envelope(&outer, sender)) + .unwrap(); + } + let requested = loop { + if let ShadowServiceEventV1::Network { + recipient, + message: NetworkMessage::RbcDagShadowCarrierRequest(reference), + } = next_event(&mut events).await + { + if recipient == 0 || recipient == 1 { + break reference; + } + } + }; + assert_eq!(requested, target.reference()); + + handle + .carrier_response( + 2, + RbcDagShadowCarrierResponse { + reference: target.reference(), + canonical_carrier: target.canonical_wire_bytes().unwrap(), + }, + ) + .unwrap(); + loop { + if let ShadowServiceEventV1::Rejected { + peer: Some(2), + error, + } = next_event(&mut events).await + { + assert!(error.contains("non-holder")); + break; + } + } + + let wrong = round_one_candidate(2, &harness.committee, 0x62); + handle + .carrier_response( + 0, + RbcDagShadowCarrierResponse { + reference: target.reference(), + canonical_carrier: wrong.canonical_wire_bytes().unwrap(), + }, + ) + .unwrap(); + loop { + if let ShadowServiceEventV1::Rejected { + peer: Some(0), + error, + } = next_event(&mut events).await + { + assert!(error.contains("ReferenceMismatch")); + break; + } + } + + let retried = timeout(Duration::from_secs(2), async { + loop { + if let ShadowServiceEventV1::Network { + message: NetworkMessage::RbcDagShadowCarrierRequest(reference), + .. + } = next_event(&mut events).await + { + break reference; + } + } + }) + .await + .unwrap(); + assert_eq!(retried, target.reference()); + + handle + .carrier_response( + 1, + RbcDagShadowCarrierResponse { + reference: target.reference(), + canonical_carrier: target.canonical_wire_bytes().unwrap(), + }, + ) + .unwrap(); + let identity = loop { + if let ShadowServiceEventV1::Delivered(identity) = next_event(&mut events).await { + break identity; + } + }; + assert_eq!(identity.author, 2); + assert_eq!(identity.round, 1); + handle.direct_delivered(identity).unwrap(); + loop { + if let ShadowServiceEventV1::Comparison(comparison) = next_event(&mut events).await { + assert_eq!(comparison, ShadowDeliveryComparisonV1::Match); + break; + } + } + let conflicting = ShadowDeliveryIdentityV1::new( + identity.author, + identity.round, + TransactionsCommitment::from_bytes([0xEE; 32]), + ); + handle.direct_delivered(conflicting).unwrap(); + loop { + if let ShadowServiceEventV1::Comparison(comparison) = next_event(&mut events).await { + assert_eq!( + comparison, + ShadowDeliveryComparisonV1::Ambiguous { + slots: vec![ShadowDeliverySlotV1 { + author: identity.author, + round: identity.round, + }], + } + ); + break; + } + } + stop(handle, events, task).await; + } + + #[tokio::test] + async fn bounded_input_reports_overload_but_shutdown_waits_for_capacity() { + let input_capacity = shadow_input_capacity(N).unwrap(); + let (sender, mut receiver) = mpsc::channel(input_capacity); + let handle = StarfishRbcDagShadowServiceHandleV1 { + sender, + max_sidecar_size: 3 + N * MAC_TAG_SIZE, + own_authority: 0, + committee_size: N, + input_capacity, + desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + invalidated_by_overload: Arc::new(Mutex::new(None)), + }; + for round in 0..input_capacity { + handle + .carrier( + 1, + RbcDagShadowCarrier { + canonical_carrier: vec![round as u8], + authentication_sidecar: Vec::new(), + }, + ) + .unwrap(); + } + assert!(matches!( + handle.carrier( + 1, + RbcDagShadowCarrier { + canonical_carrier: vec![0xFF], + authentication_sidecar: Vec::new(), + }, + ), + Err(ShadowServiceErrorV1::Overloaded { + kind: "carrier", + capacity, + }) + if capacity == input_capacity + )); + + let shutdown_handle = handle.clone(); + let shutdown = tokio::spawn(async move { shutdown_handle.shutdown().await }); + tokio::task::yield_now().await; + assert!(!shutdown.is_finished()); + while let Some(message) = receiver.recv().await { + if let ShadowServiceMessageV1::Shutdown(reply) = message { + reply.send(Ok(())).unwrap(); + break; + } + } + shutdown.await.unwrap().unwrap(); + + let (sender, _receiver) = mpsc::channel(1); + let oversized = StarfishRbcDagShadowServiceHandleV1 { + sender, + max_sidecar_size: 3 + N * MAC_TAG_SIZE, + own_authority: 0, + committee_size: N, + input_capacity: 1, + desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + invalidated_by_overload: Arc::new(Mutex::new(None)), + }; + assert!(matches!( + oversized.carrier( + 1, + RbcDagShadowCarrier { + canonical_carrier: vec![0; MAX_CARRIER_CONTENT_SIZE_V1 + 1], + authentication_sidecar: Vec::new(), + }, + ), + Err(ShadowServiceErrorV1::InputTooLarge { + field: "carrier", + .. + }) + )); + } + + #[tokio::test] + async fn sixty_validator_burst_fits_before_the_actor_drains() { + const LARGE_N: usize = 60; + let input_capacity = shadow_input_capacity(LARGE_N).unwrap(); + assert_eq!(input_capacity, SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1); + let (sender, _receiver) = mpsc::channel(input_capacity); + let invalidated = Arc::new(Mutex::new(None)); + let handle = StarfishRbcDagShadowServiceHandleV1 { + sender, + max_sidecar_size: 3 + LARGE_N * MAC_TAG_SIZE, + own_authority: 0, + committee_size: LARGE_N, + input_capacity, + desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + invalidated_by_overload: Arc::clone(&invalidated), + }; + for peer in 1..LARGE_N { + handle + .carrier( + peer as AuthorityIndex, + RbcDagShadowCarrier { + canonical_carrier: vec![peer as u8], + authentication_sidecar: Vec::new(), + }, + ) + .unwrap(); + } + for _ in 0..SHADOW_SERVICE_CONTROL_RESERVE_V1 { + handle.send(ShadowServiceMessageV1::RetryRecovery).unwrap(); + } + assert_eq!(*invalidated.lock(), None); + assert!(matches!( + shadow_input_capacity(LARGE_N + 1), + Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { .. }) + )); + } + + #[tokio::test] + async fn dropping_all_handles_stops_actor_despite_retry_timer() { + let harness = Harness::new(); + let (handle, events, task) = harness.start(0, Vec::new()); + drop(handle); + drop(events); + timeout(EVENT_TIMEOUT, task) + .await + .expect("retry timer retained a strong input sender") + .unwrap(); + } +} diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index c0652e6e..84660acf 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -20,6 +20,7 @@ use crate::{ runtime::timestamp_utc, sailfish_service::SailfishServiceMessage, starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, + starfish_rbc_dag_shadow_service::StarfishRbcDagShadowServiceHandleV1, starfish_rbc_service::{RbcLocalHeader, RbcServiceHandle}, types::{ AuthorityIndex, BlockReference, PartialSig, PartialSigKind, ProvableShard, @@ -67,6 +68,7 @@ pub struct Syncer { bls_tx: Option>, sailfish_tx: Option>, starfish_rbc_service: Option, + starfish_rbc_dag_shadow_service: Option, } pub trait SyncerSignals: Send + Sync { @@ -99,6 +101,7 @@ impl Syncer { bls_tx: Option>, sailfish_tx: Option>, starfish_rbc_service: Option, + starfish_rbc_dag_shadow_service: Option, ) -> Self { let committee_size = core.committee().len(); let own_stake = core @@ -119,6 +122,7 @@ impl Syncer { bls_tx, sailfish_tx, starfish_rbc_service, + starfish_rbc_dag_shadow_service, } } @@ -354,6 +358,18 @@ impl Syncer { *block.reference(), "RBC service selected a different local header reference" ); + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.local_header(&canonical) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + self.metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["local", "dropped"]) + .inc(); + tracing::warn!( + "Failed to enqueue non-authoritative RBC-DAG shadow carrier; comparison is invalid: {error}" + ); + } + } } if let Some(started_at) = self.proposal_wait_started_at.take() { self.metrics @@ -633,7 +649,16 @@ mod tests { assert_eq!(core.dag_state().proposal_round(), 3); assert_eq!(core.last_proposed(), 0); - let mut syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None, None); + let mut syncer = Syncer::new( + core, + false, + NoopCommitObserver, + metrics, + None, + None, + None, + None, + ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); syncer.recompute_subscriber_stake(); @@ -731,6 +756,7 @@ mod tests { None, None, None, + None, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 65ac2a8f..c5740afb 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -52,6 +52,11 @@ impl Validator { ) .map_err(|error| eyre!(error))?; let is_starfish_rbc = protocol_config.consensus_protocol.is_starfish_rbc(); + if public_config.parameters.starfish_rbc_dag_shadow && !is_starfish_rbc { + return Err(eyre!( + "Starfish-RBC-DAG shadow mode requires consensus 'starfish-rbc'" + )); + } if is_starfish_rbc { let protocol_instance = public_config .parameters @@ -195,6 +200,7 @@ impl Validator { } else { None }; + let starfish_rbc_dag_shadow_wal = private_config.starfish_rbc_dag_shadow_wal(); let (core, bls_cert_aggregator) = Core::open( block_handler, @@ -226,6 +232,7 @@ impl Validator { commit_handler, metrics.clone(), public_config.parameters.clone(), + starfish_rbc_dag_shadow_wal, partial_sig_rx, bls_cert_aggregator, bls_signer_for_service, @@ -288,6 +295,10 @@ mod smoke_tests { use crate::{ committee::Committee, config::{self, NodePrivateConfig, NodePublicConfig, Parameters}, + metrics::{ + STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, + STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, + }, prometheus, types::AuthorityIndex, }; @@ -315,16 +326,54 @@ mod smoke_tests { } } + #[tokio::test] + async fn starfish_rbc_dag_shadow_rejects_non_rbc_protocol() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_shadow = true; + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("shadow mode requires consensus 'starfish-rbc'") + })); + } + async fn run_commit_test( consensus: &str, block_authentication: Option<&str>, port_offset: u16, + ) { + run_commit_test_with_shadow(consensus, block_authentication, port_offset, false).await; + } + + async fn run_commit_test_with_shadow( + consensus: &str, + block_authentication: Option<&str>, + port_offset: u16, + starfish_rbc_dag_shadow: bool, ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); + public_config.parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; if consensus == "starfish-rbc" { public_config .parameters @@ -372,6 +421,94 @@ mod smoke_tests { ), } + if starfish_rbc_dag_shadow { + let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR + * i64::try_from(committee_size).unwrap(); + tokio::time::timeout(timeout, async { + loop { + let complete = validators.iter().all(|validator| { + let metrics = validator.metrics(); + let direct_deliveries = metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "direct"]) + .get(); + let shadow_deliveries = metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get(); + let matches = metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["match"]) + .get(); + metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1 + && metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .get() + > 0 + && direct_deliveries > 0 + && shadow_deliveries > 0 + && matches > 0 + && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 + && metrics.starfish_rbc_dag_shadow_unpaired_direct.get() + <= maximum_unpaired + && metrics.starfish_rbc_dag_shadow_unpaired_shadow.get() + <= maximum_unpaired + && metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get() + <= STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG + }); + if complete { + break; + } + time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("shadow did not durably deliver and match direct RBC before timeout"); + + for validator in &validators { + let metrics = validator.metrics(); + assert_eq!( + metrics.starfish_rbc_dag_shadow_comparison_valid.get(), + 1, + "shadow observation stream shed work while direct RBC committed" + ); + assert_eq!( + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["mismatch"]) + .get(), + 0 + ); + assert_eq!( + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["direct_only"]) + .get(), + 0 + ); + assert_eq!( + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["shadow_only"]) + .get(), + 0 + ); + assert_eq!( + metrics + .starfish_rbc_dag_shadow_delivery_comparisons_total + .with_label_values(&["ambiguous"]) + .get(), + 0 + ); + assert!(metrics.starfish_rbc_dag_shadow_unpaired_direct.get() <= maximum_unpaired); + assert!(metrics.starfish_rbc_dag_shadow_unpaired_shadow.get() <= maximum_unpaired); + assert!( + metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get() + <= STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG + ); + } + } + for v in validators { v.stop().await; } @@ -421,6 +558,11 @@ mod smoke_tests { run_commit_test("bluestreak", None, 150).await; } + #[tokio::test] + async fn starfish_rbc_dag_shadow_mac_keeps_direct_commits_live() { + run_commit_test_with_shadow("starfish-rbc", Some("mac"), 1640, true).await; + } + #[tokio::test] async fn starfish_rbc_single_validator_starts_on_current_thread_runtime() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index ed651520..ce06da66 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -72,6 +72,10 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, + /// Run the persisted, non-authoritative RBC-DAG shadow alongside + /// `starfish-rbc`. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_shadow: bool, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -105,6 +109,10 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, + /// Run the persisted, non-authoritative RBC-DAG shadow alongside + /// `starfish-rbc`. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_shadow: bool, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -160,6 +168,10 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, + /// Run the persisted, non-authoritative RBC-DAG shadow alongside + /// `starfish-rbc`. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_shadow: bool, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, /// Dissemination mode override: @@ -194,6 +206,7 @@ async fn main() -> Result<()> { byzantine_strategy, consensus: consensus_protocol, block_authentication, + starfish_rbc_dag_shadow, } => { run( authority, @@ -204,6 +217,7 @@ async fn main() -> Result<()> { byzantine_strategy, consensus_protocol, block_authentication, + starfish_rbc_dag_shadow, ) .await? } @@ -218,6 +232,7 @@ async fn main() -> Result<()> { adversarial_latency_percent, consensus: consensus_protocol, block_authentication, + starfish_rbc_dag_shadow, data_dir, base_ip, storage_backend, @@ -237,6 +252,7 @@ async fn main() -> Result<()> { adversarial_latency_percent, consensus_protocol, block_authentication, + starfish_rbc_dag_shadow, data_dir, base_ip, storage_backend, @@ -258,6 +274,7 @@ async fn main() -> Result<()> { adversarial_latency_percent, consensus: consensus_protocol, block_authentication, + starfish_rbc_dag_shadow, duration_secs, dissemination_mode, } => { @@ -268,6 +285,7 @@ async fn main() -> Result<()> { node_parameters.adversarial_latency = adversarial_latency; node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.block_authentication = block_authentication; + node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; if consensus_protocol == "starfish-rbc" { node_parameters.refresh_starfish_rbc_protocol_instance(); } @@ -412,6 +430,7 @@ async fn local_benchmark( parameters.clone() }; let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); + let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; // Create temporary directories for each validator let base_dir = PathBuf::from("local-benchmark"); @@ -498,6 +517,30 @@ async fn local_benchmark( handles.push(handle); } + if starfish_rbc_dag_shadow_expected { + let ready = tokio::time::timeout(Duration::from_secs(30), async { + loop { + if metrics_of_honest_validators + .iter() + .all(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await; + if ready.is_err() { + for abort_handle in &abort_handles { + abort_handle.abort(); + } + fs::remove_dir_all(&base_dir)?; + eyre::bail!( + "Starfish-RBC-DAG shadow did not become ready on every honest validator; benchmark was not started" + ); + } + } + // Run for specified duration tokio::select! { _ = tokio::time::sleep(Duration::from_secs(duration_secs)) => { @@ -510,6 +553,8 @@ async fn local_benchmark( metrics_of_honest_validators, reporters_of_honest_validators, duration_secs, + committee_size, + starfish_rbc_dag_shadow_expected, ); // Abort all tasks @@ -533,6 +578,8 @@ async fn local_benchmark( metrics_of_honest_validators, reporters_of_honest_validators, duration_secs, + committee_size, + starfish_rbc_dag_shadow_expected, ); fs::remove_dir_all(base_dir)?; Ok(()) @@ -550,6 +597,7 @@ async fn run( byzantine_strategy: String, consensus_protocol: String, block_authentication: Option, + starfish_rbc_dag_shadow: bool, ) -> Result<()> { tracing::info!("Starting node {authority}"); @@ -561,6 +609,9 @@ async fn run( if block_authentication.is_some() { public_config.parameters.block_authentication = block_authentication; } + if starfish_rbc_dag_shadow { + public_config.parameters.starfish_rbc_dag_shadow = true; + } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" ))?; @@ -598,6 +649,7 @@ async fn dryrun( adversarial_latency_percent: u32, consensus_protocol: String, block_authentication: Option, + starfish_rbc_dag_shadow: bool, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -640,6 +692,8 @@ async fn dryrun( node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.compress_network = compress_network; node_parameters.block_authentication = block_authentication; + node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; + ensure_starfish_rbc_protocol_instance(&consensus_protocol, &mut node_parameters); if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -689,6 +743,17 @@ async fn dryrun( Ok(()) } +fn ensure_starfish_rbc_protocol_instance( + consensus_protocol: &str, + node_parameters: &mut NodeParameters, +) { + if consensus_protocol == "starfish-rbc" + && node_parameters.starfish_rbc_protocol_instance.is_none() + { + node_parameters.refresh_starfish_rbc_protocol_instance(); + } +} + fn ipv4_add_offset(base: Ipv4Addr, offset: usize) -> Result { let offset = u32::try_from(offset).context("validator count exceeds IPv4 offset range")?; let next = u32::from(base) @@ -750,7 +815,8 @@ mod tests { use clap::Parser; - use super::{Args, Operation, ipv4_add_offset}; + use super::{Args, Operation, ensure_starfish_rbc_protocol_instance, ipv4_add_offset}; + use starfish_core::config::NodeParameters; #[test] fn ipv4_add_offset_crosses_octet_boundary() { @@ -806,12 +872,14 @@ mod tests { "starfish-rbc", "--block-authentication", "mac", + "--starfish-rbc-dag-shadow", ]) .unwrap(); let Operation::LocalBenchmark { consensus, block_authentication, + starfish_rbc_dag_shadow, .. } = args.operation else { @@ -819,5 +887,23 @@ mod tests { }; assert_eq!(consensus, "starfish-rbc"); assert_eq!(block_authentication.as_deref(), Some("mac")); + assert!(starfish_rbc_dag_shadow); + } + + #[test] + fn dry_run_starfish_rbc_configuration_gets_a_protocol_instance() { + let mut parameters = NodeParameters { + starfish_rbc_dag_shadow: true, + ..NodeParameters::default() + }; + + ensure_starfish_rbc_protocol_instance("starfish-rbc", &mut parameters); + + assert!( + parameters + .starfish_rbc_protocol_instance + .is_some_and(|instance| instance != [0; 32]) + ); + assert!(parameters.starfish_rbc_dag_shadow); } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index d08bf7d8..592fff11 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,9 +1,13 @@ # Starfish-RBC-DAG protocol design -Status: milestone-two reference codec and executable model; no runtime or safety/liveness claim +Status: milestone-three persisted, non-authoritative shadow runtime; no authoritative protocol, +safety/liveness, or performance claim -The provisional CLI name for this protocol is `starfish-rbc-dag`. It is a new protocol, not a -transport option or a version-two alias for `starfish-rbc`. +The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not +implemented. The current runtime is enabled with `--consensus starfish-rbc +--starfish-rbc-dag-shadow`; it observes the direct prototype without changing its DAG, pacemaker, +commit, or output. The eventual protocol is new, not a transport option or a version-two alias for +`starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -33,13 +37,43 @@ remain selectable outer-authentication baselines; changing that selector does no embedded RBC or consensus rules. This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish -commit rules already exist. The isolated milestone-two implementation now provides a canonical -codec plus deterministic carrier/RBC, certified-projection, decision, and crash-journal models. -Those models are not runtime integration or a proof: the composition through two clocks, two -logical projections, and frontier-based payload ordering still requires shadow execution, -additional adversarial testing, and a safety/liveness argument. Until those obligations are -discharged, `starfish-rbc-dag` must be described as an experimental reference model rather than a -proven signature-free Starfish variant. +commit rules already exist. Milestone two provides a canonical codec plus deterministic +carrier/RBC, certified-projection, decision, and crash-journal models. Milestone three adds an +opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired +direct/shadow delivery observations. The direct `starfish-rbc` service remains the only authority: +shadow admission, delivery, recovery, or failure cannot advance a proposal, mark a DAG vertex +clean, vote, commit, or order output. + +Milestone-three restart coverage is deliberately scoped to reopening the shadow actor and its WAL +against an identical recovered direct-header history. It is not a full validator crash-recovery +claim. The authoritative direct `starfish-rbc` baseline does not yet durably record its remote-slot +ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it +has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest +recovered own header dirty. Full validator restart remains fail-stop until those direct-RBC locks +are persisted and replayed; replaying only the observational shadow WAL cannot repair or safely +substitute for them. + +That isolation is logical, not physical: shadow frames share the validator's existing TCP +connections, outbound queues, bandwidth, and CPU process with direct RBC, so enabling the shadow +can perturb authoritative timing even though no shadow result is consumed by consensus. Version +one has no feature handshake. Every validator in a shadow run must use a binary that understands +the append-only shadow wire variants and the flag must be deployed committee-wide; an older peer +will reject an unknown bincode variant and may close the shared connection. Mixed-version or +partially enabled runs are not valid comparisons. + +Shadow shutdown is bounded so an observational WAL failure cannot indefinitely block validator +shutdown. If that timeout fires, the blocking worker may still hold the shadow WAL's single-writer +handle even after its async supervisor is detached. A same-process restart against that storage is +therefore forbidden until the worker has exited; process exit remains safe. A production-quality +same-process restart path needs an operating-system file lock or a fully cancellable storage task. + +The shadow runtime is not a proof or a performance implementation. It intentionally fsyncs every +accepted transition and its reference reducer clones retained model/journal history, so total CPU +work grows superlinearly with a long run. It also uses a fixed unsolicited-retention window only as +a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the +composition, resource bounds, and performance path are completed, `starfish-rbc-dag` must be +described as an experimental shadow/reference implementation rather than a proven signature-free +Starfish variant or a fair throughput baseline. The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not @@ -701,19 +735,40 @@ Hash-sorting recovered carriers is not a valid reconstruction rule. Byzantine eq arrival order determine which value a local slot-global guard selects, and a different restart order could make one honest authority appear to send conflicting phases. -The initial model and shadow prototype retain all proof-critical carrier, phase, prefix, and -consensus state for the run. Before garbage collection is enabled, the design needs a common -retirement watermark that preserves: +The proof model retains all proof-critical carrier, phase, prefix, and consensus state for the run. +The milestone-three shadow bounds newly arriving unsolicited content to a fixed recent-round +window solely to keep a faulty peer from growing an observational benchmark process without limit. +This is not a protocol-safe retirement rule: an honest INIT may be delayed longer than that under +asynchrony. Recovery of an exact already-requested value is exempt. Before authoritative garbage +collection is enabled, the design needs a common certified or committed retirement watermark that +preserves: - pending Bracha totality and header recovery; - exact self-prefix expansion from the last committed frontier; - committed-anchor reconstruction for a late validator; and - deterministic replay of local locks. -Resource bounds still required before authoritative deployment include a future carrier window, -per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, a bounded payload -runahead policy, and disk-backed recovery. Resource exhaustion is excluded from the initial proof -model but must be measured in the prototype. +Resource bounds still required before authoritative deployment include a proof-safe future and +retirement window, per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, +a bounded payload runahead policy, and checkpointed disk-backed recovery. Shadow input and output +channels are bounded and shed observational work instead of backpressuring direct consensus, but +the reference reducer's retained history and per-transition validation are not yet bounded-runtime +architecture. Resource exhaustion is excluded from the initial proof model and must be measured in +the prototype. Any run in which work is shed is invalid for direct/shadow comparison; +`starfish_rbc_dag_shadow_comparison_valid` must remain `1` for the entire measured interval. A live +pipeline does not have equal cumulative direct and shadow delivery counters at an arbitrary +instant: embedded ECHO/READY normally leaves a short shadow tail. Benchmark verification therefore +requires monotone nonzero direct, shadow, and paired-match progress, no conflict outcome, and bounds +both the current unpaired slots (`<= 4n` per validator) and the oldest unpaired round lag against the +newest current-process observation (`<= 4`). These are empirical benchmark coverage guards, not +asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as +proof of a protocol failure. + +The milestone-three actor reserves the full hard 64-entry queue so several fan-in bursts can wait +behind a synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars +and allocator overhead). It still verifies that one peer fan-in plus five local/control inputs fits; +shadow runs above 60 validators are rejected rather than silently producing an incomplete +comparison. ## 15. Safety obligations @@ -783,8 +838,10 @@ minimum it must cover: - equal committed anchors producing byte-identical output deltas; - delayed data availability followed by eventual prefix inclusion; - crash points before and after each persisted lock and outbound-carrier write; and -- once milestone three supplies the non-authoritative runtime path, shadow replay matching the - current direct RBC kernel's delivered references. +- persisted shadow-actor restart with byte-identical retransmission against an identical recovered + direct-header history, bounded overload, poisoned-tag candidate retention, exact recovery, and + paired delivery observations against the current direct RBC kernel. Full validator restart is + excluded until the authoritative direct-RBC locks are durable. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -809,7 +866,10 @@ Batching can reduce the number of separately scheduled RBC control messages, but their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout transport is added. Shadow mode also sends both direct and embedded transcripts and is a correctness -instrument, not a performance result. +instrument, not a performance result. In milestone three it additionally fsyncs each accepted +transition and validates through a clone-based reference reducer. Those costs are deliberately not +charged as protocol overhead: performance runs require incremental state transitions/checkpoints +or an equivalently durable baseline, plus separate WAL/fsync accounting. ## 19. Contained implementation milestones @@ -822,10 +882,12 @@ Every milestone is committed separately. frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable journal models; and deterministic adversarial simulations. No network or existing consensus path changes. -3. **Persisted shadow carrier path:** build and store carriers alongside the current direct +3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the current direct `starfish-rbc` service, cache the validated committee/domain identity rather than re-hashing all public keys per carrier, journal ingress and local locks, and compare embedded versus direct RBC - delivery. Direct RBC remains authoritative; shadow results never affect proposals or commits. + delivery through current-process paired observations. Direct RBC remains authoritative; shadow + results never affect proposals or commits. The reference WAL/reducer is a correctness instrument, + not yet an interpretable protocol-performance path. 4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses the current baseline. From 68a0a68c74ee149cb62504dcf8ce4856c402d537 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:34:47 +0200 Subject: [PATCH 31/62] Add autonomous Starfish-RBC-DAG shadow clock --- README.md | 44 +- crates/orchestrator/src/benchmark.rs | 91 +- crates/orchestrator/src/main.rs | 50 +- crates/orchestrator/src/measurements.rs | 550 ++++++- crates/orchestrator/src/orchestrator.rs | 16 +- crates/orchestrator/src/protocol/starfish.rs | 49 +- crates/starfish-core/src/config.rs | 68 +- crates/starfish-core/src/metrics.rs | 457 +++++- crates/starfish-core/src/net_sync.rs | 203 ++- crates/starfish-core/src/network.rs | 73 + .../src/starfish_rbc_dag_shadow.rs | 224 ++- .../src/starfish_rbc_dag_shadow_service.rs | 1371 ++++++++++++++++- crates/starfish-core/src/validator.rs | 195 ++- crates/starfish/src/main.rs | 105 +- docs/starfish-rbc-dag-protocol.md | 102 +- 15 files changed, 3401 insertions(+), 197 deletions(-) diff --git a/README.md b/README.md index 7445b861..8d6c8ded 100644 --- a/README.md +++ b/README.md @@ -50,25 +50,41 @@ Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctnes prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). **Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate logical projection. Its -canonical types, deterministic models, crash journal, and an opt-in persisted network shadow are -implemented. Run the shadow with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; direct -Starfish-RBC remains solely authoritative and shadow failures or results cannot affect proposals, -commits, or output as protocol state. Shadow traffic still shares the validator's network socket and -bandwidth, so it can perturb timing, and it must be enabled only on a homogeneous new-binary -committee; there is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` -selector is not implemented yet. The shadow uses per-transition fsync and a clone-based reference -reducer, so it is a correctness instrument, not a fair performance baseline, and carries no safety -or liveness claim. Its WAL can reopen the shadow actor against matching recovered direct headers, -but this is not full validator crash recovery: authoritative direct Starfish-RBC phase and delivery -locks are not durable yet, so that baseline remains fail-stop across process restart. The design and -proof obligations are documented in the +canonical types, deterministic models, crash journal, direct-header comparison shadow, and a +separate opt-in autonomous heartbeat carrier clock are implemented. Run the comparison shadow with +`--consensus starfish-rbc --starfish-rbc-dag-shadow`. Add +`--starfish-rbc-dag-autonomous-clock` to run an independent control-only carrier clock (prototype +heartbeat default: 250 ms); that mode deliberately does not map direct application headers to +carrier rounds or claim a direct-delivery comparison. Direct Starfish-RBC remains solely +authoritative in both modes, and shadow failures or results cannot affect proposals, commits, or +output as protocol state. Shadow traffic still shares the validator's network socket and bandwidth, +so it can perturb timing, and it must be enabled only on a homogeneous new-binary committee; there +is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` selector is not +implemented yet. The shadow uses per-transition fsync and a clone-based reference reducer, so it is +a correctness instrument, not a fair performance baseline, and carries no safety or liveness claim. +Its WAL can reopen the shadow actor, but this is not full validator crash recovery: authoritative +direct Starfish-RBC phase and delivery locks are not durable yet, so that baseline remains fail-stop +across process restart. The design and proof obligations are documented in the [protocol design](docs/starfish-rbc-dag-protocol.md). -For any shadow comparison, +For a direct-header shadow comparison, `starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded observational path was disabled or shed work and the comparison must be discarded. Healthy live production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between -the cumulative direct and shadow delivery counters. +the cumulative direct and shadow delivery counters. Autonomous runs instead require +`starfish_rbc_dag_shadow_clock_valid == 1`, durable heartbeat progress, advancing carrier rounds, +in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports +at most 60 validators in mirror mode and 20 in autonomous mode. + +An exploratory 10-validator, 60-second local run with the AWS RTT emulator, nominal 1,000 tx/s +load, MAC authentication, and a 250 ms autonomous heartbeat completed with a `VALID` clock +verdict on 2026-08-11. All validators reached carrier round 196 with zero skew and zero pending +recovery; the run recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative +direct Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 +end-to-end latency, and 0.45 MB/s average outbound bandwidth. This is a prototype continuity +result, not a fair performance comparison: the 60-second cutoff includes the local generator +warmup, and the control-only shadow still performs per-transition fsync/reference-model work while +sharing the authoritative socket. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index dc024546..f29acaf9 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -125,6 +125,31 @@ pub struct BenchmarkRunSummary { pub shadow_unpaired_shadow: usize, #[serde(default)] pub shadow_unpaired_max_round_lag: usize, + /// Whether this run used the independent, non-authoritative carrier clock + /// instead of the direct-RBC mirror comparison. + #[serde(default)] + pub shadow_autonomous_clock_enabled: bool, + /// Sticky run verdict: every validator that was ready at benchmark start + /// exposed `clock_valid == 1`, made active-window heartbeat, embedded-RBC + /// delivery, WAL, and carrier-round progress, stayed within the + /// experimental live-state/skew bounds, and supplied a valid mandatory + /// final scrape. + #[serde(default)] + pub shadow_autonomous_clock_valid: bool, + #[serde(default)] + pub shadow_autonomous_clock_valid_nodes: usize, + #[serde(default)] + pub shadow_autonomous_clock_carrier_round_min: usize, + #[serde(default)] + pub shadow_autonomous_clock_carrier_round_max: usize, + #[serde(default)] + pub shadow_autonomous_clock_phase_backlog_total: usize, + #[serde(default)] + pub shadow_autonomous_clock_admitted_authors_min: usize, + #[serde(default)] + pub shadow_autonomous_clock_admitted_stake_min: usize, + #[serde(default)] + pub shadow_autonomous_clock_buffered_authenticated_total: usize, } impl BenchmarkRunSummary { @@ -150,7 +175,15 @@ impl BenchmarkRunSummary { shadow_delivery_mismatches,shadow_delivery_ambiguous,\ shadow_wal_durable_records,shadow_pending_recovery,\ shadow_unpaired_direct,shadow_unpaired_shadow,\ - shadow_unpaired_max_round_lag" + shadow_unpaired_max_round_lag,\ + shadow_autonomous_clock_enabled,shadow_autonomous_clock_valid,\ + shadow_autonomous_clock_valid_nodes,\ + shadow_autonomous_clock_carrier_round_min,\ + shadow_autonomous_clock_carrier_round_max,\ + shadow_autonomous_clock_phase_backlog_total,\ + shadow_autonomous_clock_admitted_authors_min,\ + shadow_autonomous_clock_admitted_stake_min,\ + shadow_autonomous_clock_buffered_authenticated_total" } pub fn csv_record(&self) -> String { @@ -197,6 +230,17 @@ impl BenchmarkRunSummary { self.shadow_unpaired_direct.to_string(), self.shadow_unpaired_shadow.to_string(), self.shadow_unpaired_max_round_lag.to_string(), + self.shadow_autonomous_clock_enabled.to_string(), + self.shadow_autonomous_clock_valid.to_string(), + self.shadow_autonomous_clock_valid_nodes.to_string(), + self.shadow_autonomous_clock_carrier_round_min.to_string(), + self.shadow_autonomous_clock_carrier_round_max.to_string(), + self.shadow_autonomous_clock_phase_backlog_total.to_string(), + self.shadow_autonomous_clock_admitted_authors_min + .to_string(), + self.shadow_autonomous_clock_admitted_stake_min.to_string(), + self.shadow_autonomous_clock_buffered_authenticated_total + .to_string(), ] .join(",") } @@ -777,8 +821,8 @@ pub mod test { use crate::settings::Settings; use super::{ - BenchmarkParametersGeneric, CommitteeScalingPlan, LatencyThroughputSweepPlan, - ProtocolParameters, StabilityOutage, + BenchmarkParametersGeneric, BenchmarkRunSummary, CommitteeScalingPlan, + LatencyThroughputSweepPlan, ProtocolParameters, StabilityOutage, }; /// Mock benchmark type for unit tests. @@ -806,6 +850,47 @@ pub mod test { type TestBenchmarkParameters = BenchmarkParametersGeneric; + #[test] + fn benchmark_csv_includes_autonomous_clock_verdict_and_state() { + let summary = BenchmarkRunSummary { + shadow_autonomous_clock_enabled: true, + shadow_autonomous_clock_valid: true, + shadow_autonomous_clock_valid_nodes: 4, + shadow_autonomous_clock_carrier_round_min: 10, + shadow_autonomous_clock_carrier_round_max: 12, + shadow_autonomous_clock_phase_backlog_total: 3, + shadow_autonomous_clock_admitted_authors_min: 3, + shadow_autonomous_clock_admitted_stake_min: 7, + shadow_autonomous_clock_buffered_authenticated_total: 2, + ..BenchmarkRunSummary::default() + }; + let headers = BenchmarkRunSummary::csv_header() + .split(',') + .map(str::trim) + .collect::>(); + let record = summary.csv_record(); + let values = record.split(',').collect::>(); + assert_eq!(headers.len(), values.len()); + + for (header, expected) in [ + ("shadow_autonomous_clock_enabled", "true"), + ("shadow_autonomous_clock_valid", "true"), + ("shadow_autonomous_clock_valid_nodes", "4"), + ("shadow_autonomous_clock_carrier_round_min", "10"), + ("shadow_autonomous_clock_carrier_round_max", "12"), + ("shadow_autonomous_clock_phase_backlog_total", "3"), + ("shadow_autonomous_clock_admitted_authors_min", "3"), + ("shadow_autonomous_clock_admitted_stake_min", "7"), + ("shadow_autonomous_clock_buffered_authenticated_total", "2"), + ] { + let index = headers + .iter() + .position(|candidate| *candidate == header) + .unwrap(); + assert_eq!(values[index], expected, "column {header}"); + } + } + #[test] fn latency_throughput_sweep_switches_to_fine_grained_steps() { let plan = LatencyThroughputSweepPlan::new( diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index eff22b62..1c30ea52 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -63,15 +63,32 @@ pub struct Opts { #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, - /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative shadow. + /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative + /// shadow. #[clap(long, global = true)] starfish_rbc_dag_shadow: bool, + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + #[clap(long, global = true)] + starfish_rbc_dag_autonomous_clock: bool, + + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT", global = true)] + starfish_rbc_dag_heartbeat_interval_ms: Option, + /// The type of operation to run. #[clap(subcommand)] operation: Operation, } +#[derive(Clone, Copy, Debug)] +struct StarfishRbcDagOverrides { + shadow: bool, + autonomous_clock: bool, + heartbeat_interval_ms: Option, +} + /// The type of operation to run. #[derive(Parser, Debug)] #[clap(rename_all = "kebab-case")] @@ -856,7 +873,7 @@ fn load_benchmark_configs( compress_network: Option, bls_workers: Option, block_authentication: &Option, - starfish_rbc_dag_shadow: bool, + starfish_rbc_dag: StarfishRbcDagOverrides, ) -> eyre::Result<(NodeParameters, ClientParameters)> { let mut node_parameters = match &settings.node_parameters_path { Some(path) => NodeParameters::load(path).wrap_err("Failed to load node's parameters")?, @@ -867,9 +884,15 @@ fn load_benchmark_configs( if block_authentication.is_some() { node_parameters.block_authentication = block_authentication.clone(); } - if starfish_rbc_dag_shadow { + if starfish_rbc_dag.shadow { node_parameters.starfish_rbc_dag_shadow = true; } + if starfish_rbc_dag.autonomous_clock { + node_parameters.starfish_rbc_dag_autonomous_clock = true; + } + if let Some(interval_ms) = starfish_rbc_dag.heartbeat_interval_ms { + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1050,7 +1073,11 @@ async fn run( .wrap_err("Failed to crate testbed")?; let block_authentication = opts.block_authentication.clone(); - let starfish_rbc_dag_shadow = opts.starfish_rbc_dag_shadow; + let starfish_rbc_dag = StarfishRbcDagOverrides { + shadow: opts.starfish_rbc_dag_shadow, + autonomous_clock: opts.starfish_rbc_dag_autonomous_clock, + heartbeat_interval_ms: opts.starfish_rbc_dag_heartbeat_interval_ms, + }; match opts.operation { Operation::Testbed { action } => match action { // Display the current status of the testbed. @@ -1248,7 +1275,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, + starfish_rbc_dag, )?; display::newline(); @@ -1416,7 +1443,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, + starfish_rbc_dag, )?; display::newline(); @@ -1624,7 +1651,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, + starfish_rbc_dag, )?; display::newline(); @@ -1791,7 +1818,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, + starfish_rbc_dag, )?; display::newline(); @@ -1998,7 +2025,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, + starfish_rbc_dag, )?; display::newline(); @@ -2345,6 +2372,9 @@ mod tests { "--block-authentication", "mac", "--starfish-rbc-dag-shadow", + "--starfish-rbc-dag-autonomous-clock", + "--starfish-rbc-dag-heartbeat-interval-ms", + "125", "--protocols", "starfish-rbc", ]) @@ -2352,6 +2382,8 @@ mod tests { assert_eq!(opts.block_authentication.as_deref(), Some("mac")); assert!(opts.starfish_rbc_dag_shadow); + assert!(opts.starfish_rbc_dag_autonomous_clock); + assert_eq!(opts.starfish_rbc_dag_heartbeat_interval_ms, Some(125)); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); }; diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index d4a78d4d..945900ab 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -16,7 +16,10 @@ use prettytable::{Table, row}; use prometheus_parse::Scrape; use serde::{Deserialize, Serialize}; use starfish_core::metrics::{ - STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, + STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR, + STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR, + STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, + STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, }; use crate::{ @@ -30,6 +33,28 @@ type BucketId = String; /// The identifier of a measurement type. type Label = String; +pub(crate) const SHADOW_COMPARISON_VALID_METRIC: &str = "starfish_rbc_dag_shadow_comparison_valid"; +pub(crate) const SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC: &str = "starfish_rbc_dag_shadow_clock_valid"; + +/// Select the validity gauge for the configured observational mode. Mirror +/// mode compares direct and embedded RBC deliveries; autonomous mode has no +/// one-to-one direct stream and therefore owns a separate clock verdict. +pub(crate) fn shadow_validity_metric(parameters: &BenchmarkParameters) -> Option<&'static str> { + if parameters.consensus_protocol != "starfish-rbc" + || !parameters.node_parameters.starfish_rbc_dag_shadow + { + return None; + } + + Some( + if parameters.node_parameters.starfish_rbc_dag_autonomous_clock { + SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC + } else { + SHADOW_COMPARISON_VALID_METRIC + }, + ) +} + /// A snapshot measurement at a given time. #[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq)] pub struct Measurement { @@ -279,6 +304,12 @@ impl Measurement { "starfish_rbc_dag_shadow_wal_replayed_batches" | "starfish_rbc_dag_shadow_pending_recovery" | "starfish_rbc_dag_shadow_comparison_valid" + | "starfish_rbc_dag_shadow_clock_valid" + | "starfish_rbc_dag_shadow_carrier_round" + | "starfish_rbc_dag_shadow_phase_backlog" + | "starfish_rbc_dag_shadow_admitted_authors" + | "starfish_rbc_dag_shadow_admitted_stake" + | "starfish_rbc_dag_shadow_buffered_authenticated" | "starfish_rbc_dag_shadow_unpaired_direct" | "starfish_rbc_dag_shadow_unpaired_shadow" | "starfish_rbc_dag_shadow_unpaired_max_round_lag" @@ -412,6 +443,9 @@ impl MeasurementsCollection { /// shadow scrape. Appending an explicit invalid observation prevents an /// earlier successful scrape from being mistaken for fresh final evidence. pub fn mark_shadow_final_scrape_missing(&mut self, scraper_id: ScraperId) { + let Some(validity_metric) = shadow_validity_metric(&self.parameters) else { + return; + }; let timestamp = self .data .values() @@ -424,7 +458,7 @@ impl MeasurementsCollection { self.synthetic_only_scrapers.insert(scraper_id); } self.data - .entry("starfish_rbc_dag_shadow_comparison_valid".to_owned()) + .entry(validity_metric.to_owned()) .or_default() .entry(scraper_id) .or_default() @@ -529,6 +563,14 @@ impl MeasurementsCollection { .sum() } + fn min_latest_scalar_as_usize(&self, label: &str) -> usize { + self.latest_measurements(label) + .into_iter() + .map(|measurement| measurement.scalar.max(0.0) as usize) + .min() + .unwrap_or_default() + } + /// Sum scalar Prometheus counter increments across scrapers and resets. fn sum_scalar_counter_increments(&self, label: &str) -> usize { self.data @@ -556,6 +598,27 @@ impl MeasurementsCollection { self.data.get(label)?.get(&scraper_id).map(Vec::as_slice) } + fn active_window_series(&self, label: &str, scraper_id: ScraperId) -> Option<&[Measurement]> { + let series = self.scraper_series(label, scraper_id)?; + if !series + .windows(2) + .all(|window| window[1].timestamp >= window[0].timestamp) + { + return None; + } + let start = series + .iter() + .position(|measurement| !measurement.timestamp.is_zero())?; + let maximum_timestamp = series.iter().map(Measurement::timestamp).max()?; + let end = series + .iter() + .position(|measurement| measurement.timestamp == maximum_timestamp)?; + let active = series.get(start..=end)?; + let first = active.first()?; + let last = active.last()?; + (active.len() >= 2 && last.timestamp > first.timestamp).then_some(active) + } + fn gauge_always_equals(&self, label: &str, scraper_id: ScraperId, expected: f64) -> bool { self.scraper_series(label, scraper_id) .is_some_and(|series| { @@ -579,6 +642,20 @@ impl MeasurementsCollection { .is_some_and(|measurement| measurement.scalar > 0.0) } + fn scalar_counter_increased(&self, label: &str, scraper_id: ScraperId) -> bool { + let Some(series) = self.active_window_series(label, scraper_id) else { + return false; + }; + series.len() >= 2 + && series + .windows(2) + .all(|window| window[1].scalar >= window[0].scalar) + && series + .last() + .zip(series.first()) + .is_some_and(|(last, first)| last.scalar > first.scalar) + } + fn count_bucket_is_monotonic_and_positive( &self, label: &str, @@ -596,6 +673,22 @@ impl MeasurementsCollection { && values.last().is_some_and(|value| *value > 0) } + fn count_bucket_increased(&self, label: &str, scraper_id: ScraperId, bucket: &str) -> bool { + let Some(series) = self.active_window_series(label, scraper_id) else { + return false; + }; + let values = series + .iter() + .map(|measurement| measurement.count_buckets.get(bucket).copied().unwrap_or(0)) + .collect::>(); + values.len() >= 2 + && values.windows(2).all(|window| window[1] >= window[0]) + && values + .last() + .zip(values.first()) + .is_some_and(|(last, first)| last > first) + } + fn count_bucket_is_always_zero( &self, label: &str, @@ -615,6 +708,26 @@ impl MeasurementsCollection { .is_some_and(|measurement| measurement.scalar == expected) } + fn latest_scalar_greater_than(&self, label: &str, scraper_id: ScraperId, minimum: f64) -> bool { + self.scraper_series(label, scraper_id) + .and_then(|series| series.last()) + .is_some_and(|measurement| measurement.scalar > minimum) + } + + fn scalar_gauge_increased(&self, label: &str, scraper_id: ScraperId) -> bool { + self.active_window_series(label, scraper_id) + .is_some_and(|series| { + series.len() >= 2 + && series + .windows(2) + .all(|window| window[1].scalar >= window[0].scalar) + && series + .last() + .zip(series.first()) + .is_some_and(|(last, first)| last.scalar > first.scalar) + }) + } + fn gauge_always_at_most(&self, label: &str, scraper_id: ScraperId, maximum: f64) -> bool { self.scraper_series(label, scraper_id) .is_some_and(|series| { @@ -860,8 +973,14 @@ impl MeasurementsCollection { } }) .collect(); - let shadow_comparison_enabled = self.parameters.consensus_protocol == "starfish-rbc" + let shadow_enabled = self.parameters.consensus_protocol == "starfish-rbc" && self.parameters.node_parameters.starfish_rbc_dag_shadow; + let shadow_autonomous_clock_enabled = shadow_enabled + && self + .parameters + .node_parameters + .starfish_rbc_dag_autonomous_clock; + let shadow_comparison_enabled = shadow_enabled && !shadow_autonomous_clock_enabled; let shadow_valid_scrapers = self .data .get("starfish_rbc_dag_shadow_comparison_valid") @@ -975,6 +1094,103 @@ impl MeasurementsCollection { && every_shadow_scraper_has_coverage && shadow_delivery_mismatches == 0 && shadow_delivery_ambiguous == 0; + let shadow_autonomous_clock_valid_scrapers = self + .data + .get(SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC) + .map(|by_scraper| { + by_scraper + .keys() + .copied() + .filter(|scraper_id| { + self.gauge_always_equals( + SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC, + *scraper_id, + 1.0, + ) + }) + .collect::>() + }) + .unwrap_or_default(); + let shadow_autonomous_clock_valid_nodes = shadow_autonomous_clock_valid_scrapers.len(); + let ( + shadow_autonomous_clock_carrier_round_min, + shadow_autonomous_clock_carrier_round_max, + shadow_autonomous_clock_phase_backlog_total, + shadow_autonomous_clock_admitted_authors_min, + shadow_autonomous_clock_admitted_stake_min, + shadow_autonomous_clock_buffered_authenticated_total, + ) = if shadow_autonomous_clock_enabled { + ( + self.min_latest_scalar_as_usize("starfish_rbc_dag_shadow_carrier_round"), + self.max_result("starfish_rbc_dag_shadow_carrier_round", |measurement| { + measurement.scalar.max(0.0) as usize + }), + self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_phase_backlog"), + self.min_latest_scalar_as_usize("starfish_rbc_dag_shadow_admitted_authors"), + self.min_latest_scalar_as_usize("starfish_rbc_dag_shadow_admitted_stake"), + self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_buffered_authenticated"), + ) + } else { + (0, 0, 0, 0, 0, 0) + }; + let autonomous_phase_backlog_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR + .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); + let autonomous_buffered_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR + .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); + let every_autonomous_scraper_has_progress = + shadow_autonomous_clock_valid_scrapers + .iter() + .all(|scraper_id| { + self.count_bucket_increased( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + "heartbeat,accepted", + ) && self.count_bucket_increased( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + "delivery,shadow", + ) && self.scalar_counter_increased( + "starfish_rbc_dag_shadow_wal_durable_batches_total", + *scraper_id, + ) && self.scalar_counter_increased( + "starfish_rbc_dag_shadow_wal_durable_records_total", + *scraper_id, + ) && self.scalar_gauge_increased( + "starfish_rbc_dag_shadow_carrier_round", + *scraper_id, + ) && self.latest_scalar_greater_than( + "starfish_rbc_dag_shadow_carrier_round", + *scraper_id, + 1.0, + ) && self.latest_scalar_equals( + "starfish_rbc_dag_shadow_pending_recovery", + *scraper_id, + 0.0, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_phase_backlog", + *scraper_id, + autonomous_phase_backlog_bound as f64, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_admitted_authors", + *scraper_id, + self.parameters.nodes as f64, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_admitted_stake", + *scraper_id, + f64::MAX, + ) && self.gauge_always_at_most( + "starfish_rbc_dag_shadow_buffered_authenticated", + *scraper_id, + autonomous_buffered_bound as f64, + ) + }); + let shadow_autonomous_clock_valid = shadow_autonomous_clock_enabled + && expected_shadow_nodes != 0 + && shadow_autonomous_clock_valid_nodes == expected_shadow_nodes + && every_autonomous_scraper_has_progress + && shadow_autonomous_clock_carrier_round_max + .saturating_sub(shadow_autonomous_clock_carrier_round_min) + <= usize::try_from(STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG).unwrap_or(usize::MAX); BenchmarkRunSummary { protocol: self.parameters.consensus_protocol.clone(), @@ -1021,6 +1237,15 @@ impl MeasurementsCollection { shadow_unpaired_direct, shadow_unpaired_shadow, shadow_unpaired_max_round_lag, + shadow_autonomous_clock_enabled, + shadow_autonomous_clock_valid, + shadow_autonomous_clock_valid_nodes, + shadow_autonomous_clock_carrier_round_min, + shadow_autonomous_clock_carrier_round_max, + shadow_autonomous_clock_phase_backlog_total, + shadow_autonomous_clock_admitted_authors_min, + shadow_autonomous_clock_admitted_stake_min, + shadow_autonomous_clock_buffered_authenticated_total, } } @@ -1093,6 +1318,28 @@ impl MeasurementsCollection { ) ]); } + if summary.shadow_autonomous_clock_enabled { + table.add_row(row![ + b->"RBC-DAG autonomous clock:", + format!( + "valid={} ({}/{} validators), carrier rounds={}..={}, embedded deliveries={}, phase backlog={}, \ + admitted authors/stake min={}/{}, buffered authenticated={}, WAL records={}, \ + pending recovery={}", + summary.shadow_autonomous_clock_valid, + summary.shadow_autonomous_clock_valid_nodes, + summary.ready_nodes_at_boot, + summary.shadow_autonomous_clock_carrier_round_min, + summary.shadow_autonomous_clock_carrier_round_max, + summary.shadow_deliveries, + summary.shadow_autonomous_clock_phase_backlog_total, + summary.shadow_autonomous_clock_admitted_authors_min, + summary.shadow_autonomous_clock_admitted_stake_min, + summary.shadow_autonomous_clock_buffered_authenticated_total, + summary.shadow_wal_durable_records, + summary.shadow_pending_recovery, + ) + ]); + } table.add_row(row![ b->"End-to-end latency:", format!( @@ -1177,6 +1424,12 @@ mod test { parameters } + fn autonomous_shadow_benchmark_parameters(nodes: usize) -> BenchmarkParameters { + let mut parameters = shadow_benchmark_parameters(nodes); + parameters.node_parameters.starfish_rbc_dag_autonomous_clock = true; + parameters + } + #[allow(clippy::too_many_arguments)] fn add_shadow_snapshot( collection: &mut MeasurementsCollection, @@ -1189,7 +1442,7 @@ mod test { direct_only: usize, shadow_only: usize, ambiguous: usize, - wal_durable_records: f64, + wal_durable_records: usize, ) { collection.add( scraper_id, @@ -1230,8 +1483,8 @@ mod test { scraper_id, "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), Measurement { - count: wal_durable_records as usize, - scalar: wal_durable_records, + count: wal_durable_records, + scalar: wal_durable_records as f64, ..Measurement::default() }, ); @@ -1269,6 +1522,95 @@ mod test { } } + #[allow(clippy::too_many_arguments)] + fn add_autonomous_clock_snapshot( + collection: &mut MeasurementsCollection, + scraper_id: usize, + clock_valid: f64, + carrier_round: usize, + phase_backlog: usize, + admitted_authors: usize, + admitted_stake: usize, + buffered_authenticated: usize, + wal_durable_records: usize, + ) { + let timestamp = Duration::from_secs(carrier_round as u64); + for (label, value) in [ + ("starfish_rbc_dag_shadow_clock_valid", clock_valid), + ( + "starfish_rbc_dag_shadow_carrier_round", + carrier_round as f64, + ), + ( + "starfish_rbc_dag_shadow_phase_backlog", + phase_backlog as f64, + ), + ( + "starfish_rbc_dag_shadow_admitted_authors", + admitted_authors as f64, + ), + ( + "starfish_rbc_dag_shadow_admitted_stake", + admitted_stake as f64, + ), + ( + "starfish_rbc_dag_shadow_buffered_authenticated", + buffered_authenticated as f64, + ), + ] { + collection.add( + scraper_id, + label.to_owned(), + Measurement { + timestamp, + scalar: value, + ..Measurement::default() + }, + ); + } + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_inputs_total".to_owned(), + Measurement { + timestamp, + count_buckets: HashMap::from([ + ("heartbeat,accepted".to_owned(), wal_durable_records), + ("delivery,shadow".to_owned(), wal_durable_records), + ]), + count: wal_durable_records.saturating_mul(2), + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_wal_durable_batches_total".to_owned(), + Measurement { + timestamp, + count: wal_durable_records, + scalar: wal_durable_records as f64, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), + Measurement { + timestamp, + count: wal_durable_records, + scalar: wal_durable_records as f64, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_pending_recovery".to_owned(), + Measurement { + timestamp, + ..Measurement::default() + }, + ); + } + #[test] fn average_latency() { let data = Measurement { @@ -1489,6 +1831,98 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 assert_eq!(summary.shadow_unpaired_max_round_lag, 1); } + #[test] + fn prometheus_parse_preserves_autonomous_clock_state() { + let report = r#" +# TYPE benchmark_duration counter +benchmark_duration 30 +# TYPE starfish_rbc_dag_shadow_clock_valid gauge +starfish_rbc_dag_shadow_clock_valid 1 +# TYPE starfish_rbc_dag_shadow_carrier_round gauge +starfish_rbc_dag_shadow_carrier_round 12 +# TYPE starfish_rbc_dag_shadow_phase_backlog gauge +starfish_rbc_dag_shadow_phase_backlog 3 +# TYPE starfish_rbc_dag_shadow_admitted_authors gauge +starfish_rbc_dag_shadow_admitted_authors 4 +# TYPE starfish_rbc_dag_shadow_admitted_stake gauge +starfish_rbc_dag_shadow_admitted_stake 7 +# TYPE starfish_rbc_dag_shadow_buffered_authenticated gauge +starfish_rbc_dag_shadow_buffered_authenticated 2 +"#; + + let measurements = Measurement::from_prometheus::(report); + for (label, expected) in [ + ("starfish_rbc_dag_shadow_clock_valid", 1.0), + ("starfish_rbc_dag_shadow_carrier_round", 12.0), + ("starfish_rbc_dag_shadow_phase_backlog", 3.0), + ("starfish_rbc_dag_shadow_admitted_authors", 4.0), + ("starfish_rbc_dag_shadow_admitted_stake", 7.0), + ("starfish_rbc_dag_shadow_buffered_authenticated", 2.0), + ] { + assert_eq!(measurements[label].scalar, expected, "metric {label}"); + } + } + + #[test] + fn autonomous_clock_has_a_distinct_sticky_summary() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 1, 2, 7, 1, 5); + add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 8, 1, 2, 6, 1, 6); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 3, 2, 7, 2, 10); + add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 10, 4, 2, 6, 3, 11); + + let summary = collection.benchmark_run_summary(); + assert!(!summary.shadow_comparison_enabled); + assert!(!summary.shadow_comparison_valid); + assert!(summary.shadow_autonomous_clock_enabled); + assert!(summary.shadow_autonomous_clock_valid); + assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 2); + assert_eq!(summary.shadow_autonomous_clock_carrier_round_min, 10); + assert_eq!(summary.shadow_autonomous_clock_carrier_round_max, 12); + assert_eq!(summary.shadow_autonomous_clock_phase_backlog_total, 7); + assert_eq!(summary.shadow_autonomous_clock_admitted_authors_min, 2); + assert_eq!(summary.shadow_autonomous_clock_admitted_stake_min, 6); + assert_eq!( + summary.shadow_autonomous_clock_buffered_authenticated_total, + 5 + ); + assert_eq!(summary.shadow_wal_durable_records, 21); + + // A later healthy scrape must not erase an earlier invalid verdict. + add_autonomous_clock_snapshot(&mut collection, 0, 0.0, 13, 0, 3, 7, 0, 12); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 14, 0, 3, 7, 0, 13); + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 1); + assert!(!summary.shadow_autonomous_clock_valid); + } + + #[test] + fn missing_final_autonomous_scrape_invalidates_only_clock_verdict() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 7, 0, 5); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 0, 1, 7, 0, 10); + assert!( + collection + .benchmark_run_summary() + .shadow_autonomous_clock_valid + ); + + collection.mark_shadow_final_scrape_missing(0); + + let validity = collection + .scraper_series("starfish_rbc_dag_shadow_clock_valid", 0) + .unwrap(); + assert_eq!(validity.last().unwrap().scalar_value(), 0.0); + assert!( + collection + .scraper_series("starfish_rbc_dag_shadow_comparison_valid", 0) + .is_none() + ); + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 0); + assert!(!summary.shadow_autonomous_clock_valid); + } + #[test] fn shadow_verdict_remains_invalid_after_historical_failure_and_counter_reset() { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); @@ -1497,8 +1931,8 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 // comparison category. The second scrape deliberately looks clean, // including reset comparison counters, so a latest-value-only verdict // would incorrectly accept the run. - add_shadow_snapshot(&mut collection, 0, 0.0, 1, 1, 1, 1, 1, 1, 1, 1.0); - add_shadow_snapshot(&mut collection, 0, 1.0, 2, 2, 2, 0, 0, 0, 0, 2.0); + add_shadow_snapshot(&mut collection, 0, 0.0, 1, 1, 1, 1, 1, 1, 1, 1); + add_shadow_snapshot(&mut collection, 0, 1.0, 2, 2, 2, 0, 0, 0, 0, 2); assert!(!collection.gauge_always_equals( "starfish_rbc_dag_shadow_comparison_valid", @@ -1519,10 +1953,100 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 assert!(!summary.shadow_comparison_valid); } + #[test] + fn autonomous_clock_valid_gauge_without_durable_progress_is_invalid() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); + + let summary = collection.benchmark_run_summary(); + assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 1); + assert!(!summary.shadow_autonomous_clock_valid); + } + + #[test] + fn autonomous_clock_warmup_only_progress_is_invalid() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 2, 0, 1, 1, 0, 3); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 4, 0, 1, 1, 0, 6); + for by_scraper in collection.data.values_mut() { + for measurement in by_scraper.get_mut(&0).into_iter().flatten() { + measurement.timestamp = Duration::ZERO; + } + } + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 4, 0, 1, 1, 0, 6); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 4, 0, 1, 1, 0, 6); + for by_scraper in collection.data.values_mut() { + if let Some(last) = by_scraper.get_mut(&0).and_then(|series| series.last_mut()) { + last.timestamp = Duration::from_secs(5); + } + } + + assert!( + !collection + .benchmark_run_summary() + .shadow_autonomous_clock_valid + ); + } + + #[test] + fn autonomous_clock_verdict_requires_in_window_rbc_delivery() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 10, 0, 1, 1, 0, 6); + collection + .data + .get_mut("starfish_rbc_dag_shadow_inputs_total") + .and_then(|by_scraper| by_scraper.get_mut(&0)) + .and_then(|series| series.last_mut()) + .unwrap() + .count_buckets + .insert("delivery,shadow".to_owned(), 3); + + assert!( + !collection + .benchmark_run_summary() + .shadow_autonomous_clock_valid + ); + } + + #[test] + fn autonomous_clock_verdict_rejects_observed_round_rollback() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 1, 0, 1, 1, 0, 4); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 9, 0, 1, 1, 0, 5); + + assert!( + !collection + .benchmark_run_summary() + .shadow_autonomous_clock_valid + ); + } + + #[test] + fn autonomous_clock_verdict_rejects_benchmark_timestamp_reset() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 10, 0, 1, 1, 0, 5); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 0, 1, 1, 0, 7); + for by_scraper in collection.data.values_mut() { + if let Some(last) = by_scraper.get_mut(&0).and_then(|series| series.last_mut()) { + last.timestamp = Duration::from_secs(1); + } + } + + assert!( + !collection + .benchmark_run_summary() + .shadow_autonomous_clock_valid + ); + } + #[test] fn missing_final_shadow_scrape_invalidates_a_previously_valid_snapshot() { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); + add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8); assert!(collection.benchmark_run_summary().shadow_comparison_valid); collection.mark_shadow_final_scrape_missing(0); @@ -1549,8 +2073,8 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 #[test] fn shadow_verdict_requires_delivery_and_wal_coverage_from_every_scraper() { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(2)); - add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); - add_shadow_snapshot(&mut collection, 1, 1.0, 0, 0, 0, 0, 0, 0, 0, 0.0); + add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8); + add_shadow_snapshot(&mut collection, 1, 1.0, 0, 0, 0, 0, 0, 0, 0, 0); let summary = collection.benchmark_run_summary(); assert_eq!(summary.shadow_comparison_valid_nodes, 2); @@ -1576,7 +2100,7 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 0, 0, 0, - 8.0, + 8, ); add_shadow_backlog_snapshot( &mut collection, @@ -1598,7 +2122,7 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 fn shadow_verdict_rejects_excessive_or_old_unpaired_work() { for (unpaired_direct, unpaired_shadow, max_round_lag) in [(5, 0, 1), (0, 5, 1), (1, 0, 5)] { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot(&mut collection, 0, 1.0, 8, 7, 7, 0, 0, 0, 0, 8.0); + add_shadow_snapshot(&mut collection, 0, 1.0, 8, 7, 7, 0, 0, 0, 0, 8); add_shadow_backlog_snapshot( &mut collection, 0, diff --git a/crates/orchestrator/src/orchestrator.rs b/crates/orchestrator/src/orchestrator.rs index 6ce99bd4..70871516 100644 --- a/crates/orchestrator/src/orchestrator.rs +++ b/crates/orchestrator/src/orchestrator.rs @@ -25,7 +25,7 @@ use crate::{ error::{SshError, TestbedError, TestbedResult}, faults::{CrashRecoverySchedule, FaultsType}, logs::LogsAnalyzer, - measurements::{Measurement, MeasurementsCollection}, + measurements::{Measurement, MeasurementsCollection, shadow_validity_metric}, monitor::Monitor, protocol::{ProtocolCommands, ProtocolMetrics}, settings::Settings, @@ -1098,8 +1098,8 @@ impl Orchestrator

{ .filter(|node| !killed_node_ids.contains(&node.id)) .filter_map(|node| node_indices.get(&node.id).copied()) .collect(); - let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" - && parameters.node_parameters.starfish_rbc_dag_shadow; + let shadow_final_validity_metric = shadow_validity_metric(parameters); + let shadow_final_scrape_required = shadow_final_validity_metric.is_some(); let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1224,7 +1224,9 @@ impl Orchestrator

{ continue; }; let parsed = Measurement::from_prometheus::

(stdout); - if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { + if shadow_final_validity_metric + .is_some_and(|metric| parsed.contains_key(metric)) + { fresh_final_shadow_scrapers.insert(i); } for (label, measurement) in parsed { @@ -1430,8 +1432,8 @@ impl Orchestrator

{ .filter(|node| !killed_node_ids.contains(&node.id)) .filter_map(|node| node_indices.get(&node.id).copied()) .collect(); - let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" - && parameters.node_parameters.starfish_rbc_dag_shadow; + let shadow_final_validity_metric = shadow_validity_metric(parameters); + let shadow_final_scrape_required = shadow_final_validity_metric.is_some(); let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1845,7 +1847,7 @@ impl Orchestrator

{ continue; }; let parsed = Measurement::from_prometheus::

(stdout); - if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { + if shadow_final_validity_metric.is_some_and(|metric| parsed.contains_key(metric)) { fresh_final_shadow_scrapers.insert(i); } for (label, measurement) in parsed { diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index e5bcdf1f..8b3bc56c 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -19,7 +19,10 @@ use super::{ BINARY_PATH, METRICS_CURL_CONNECT_TIMEOUT_SECS, METRICS_CURL_MAX_TIME_SECS, ProtocolCommands, ProtocolMetrics, ProtocolParameters, }; -use crate::{benchmark::BenchmarkParameters, client::Instance, settings::Settings}; +use crate::{ + benchmark::BenchmarkParameters, client::Instance, measurements::shadow_validity_metric, + settings::Settings, +}; #[derive(Clone, Serialize, Deserialize, Default)] #[serde(transparent)] @@ -275,6 +278,8 @@ impl ProtocolMetrics for StarfishProtocol { .collect(); } + let validity_metric = shadow_validity_metric(parameters) + .expect("Starfish-RBC shadow readiness has an active validity metric"); self.nodes_metrics_path(instances, parameters) .into_iter() .map(|(instance, path)| { @@ -284,7 +289,7 @@ impl ProtocolMetrics for StarfishProtocol { "curl --silent --show-error --fail --compressed --connect-timeout \ {METRICS_CURL_CONNECT_TIMEOUT_SECS} --max-time \ {METRICS_CURL_MAX_TIME_SECS} {path} | grep -Eq \ - '^starfish_rbc_dag_shadow_comparison_valid(\\{{[^}}]*\\}})? \ + '^{validity_metric}(\\{{[^}}]*\\}})? \ 1(\\.0)?$'" ), ) @@ -318,6 +323,9 @@ impl StarfishProtocol { // not let it make Sailfish++ or another comparison member fail // validator configuration. node_parameters.starfish_rbc_dag_shadow = false; + node_parameters.starfish_rbc_dag_autonomous_clock = false; + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = + config::node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(); } node_parameters } @@ -354,12 +362,14 @@ impl StarfishProtocol { mod tests { use super::{ProtocolMetrics, StarfishNodeParameters, StarfishProtocol}; use crate::{benchmark::BenchmarkParameters, client::Instance}; - use starfish_core::config::NodeParameters; + use starfish_core::config::{NodeParameters, node_defaults}; #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, + starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }); let parameters = @@ -370,12 +380,16 @@ mod tests { .is_some_and(|instance| instance != [0; 32]) ); assert!(parameters.starfish_rbc_dag_shadow); + assert!(parameters.starfish_rbc_dag_autonomous_clock); + assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } #[test] fn non_rbc_genesis_does_not_need_a_protocol_instance() { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, + starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }); let parameters = @@ -385,6 +399,15 @@ mod tests { !parameters.starfish_rbc_dag_shadow, "a global shadow flag must not leak into non-RBC comparison members" ); + assert!( + !parameters.starfish_rbc_dag_autonomous_clock, + "a global autonomous-clock flag must not leak into non-RBC comparison members" + ); + assert_eq!( + parameters.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), + "a global heartbeat override must not leak into non-RBC comparison members" + ); } #[test] @@ -405,6 +428,26 @@ mod tests { assert!(command.contains("grep -Eq")); } + #[test] + fn autonomous_shadow_readiness_waits_for_clock_verdict() { + let protocol = StarfishProtocol { + working_dir: std::path::PathBuf::from("benchmark"), + }; + let mut parameters = BenchmarkParameters::new_for_tests(); + parameters.consensus_protocol = "starfish-rbc".to_owned(); + parameters.node_parameters.starfish_rbc_dag_shadow = true; + parameters.node_parameters.starfish_rbc_dag_autonomous_clock = true; + let command = protocol + .nodes_readiness_command(vec![Instance::new_for_test("1".into())], ¶meters) + .pop() + .unwrap() + .1; + + assert!(command.contains("starfish_rbc_dag_shadow_clock_valid")); + assert!(!command.contains("starfish_rbc_dag_shadow_comparison_valid")); + assert!(command.contains("grep -Eq")); + } + #[test] fn split_authority_load_preserves_total_load() { let nodes = 4; diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 63b341e1..12c3a161 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -72,6 +72,15 @@ pub struct NodeParameters { /// observational only and cannot affect the DAG, pacemaker, or commits. #[serde(default)] pub starfish_rbc_dag_shadow: bool, + /// Let the non-authoritative Starfish-RBC-DAG shadow create an autonomous + /// optimistic carrier clock. This remains experimental and requires + /// `starfish_rbc_dag_shadow`. + #[serde(default)] + pub starfish_rbc_dag_autonomous_clock: bool, + /// Maximum interval between autonomous Starfish-RBC-DAG heartbeat + /// carriers. The value is ignored unless the autonomous clock is enabled. + #[serde(default = "node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms")] + pub starfish_rbc_dag_heartbeat_interval_ms: u64, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -122,6 +131,10 @@ pub mod node_defaults { 5 } + pub fn default_starfish_rbc_dag_heartbeat_interval_ms() -> u64 { + 250 + } + pub fn default_causal_push_shard_round_lag() -> RoundNumber { 0 } @@ -148,6 +161,9 @@ impl Default for NodeParameters { block_authentication: None, starfish_rbc_protocol_instance: None, starfish_rbc_dag_shadow: false, + starfish_rbc_dag_autonomous_clock: false, + starfish_rbc_dag_heartbeat_interval_ms: + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), @@ -385,19 +401,31 @@ impl NodePrivateConfig { pub fn starfish_rbc_dag_shadow_wal(&self) -> PathBuf { self.storage_path.join("starfish-rbc-dag-shadow-v1.wal") } + + pub fn starfish_rbc_dag_autonomous_clock_wal(&self) -> PathBuf { + self.storage_path + .join("starfish-rbc-dag-autonomous-clock-v1.wal") + } } impl ImportExport for NodePrivateConfig {} #[cfg(test)] mod tests { - use super::NodeParameters; + use std::path::Path; + + use super::{NodeParameters, NodePrivateConfig, node_defaults}; #[test] fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { let mut parameters: NodeParameters = serde_yaml::from_str("{}").unwrap(); assert_eq!(parameters.starfish_rbc_protocol_instance, None); assert!(!parameters.starfish_rbc_dag_shadow); + assert!(!parameters.starfish_rbc_dag_autonomous_clock); + assert_eq!( + parameters.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() + ); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); assert_ne!(protocol_instance, [0; 32]); @@ -409,6 +437,44 @@ mod tests { Some(protocol_instance) ); assert!(!decoded.starfish_rbc_dag_shadow); + assert!(!decoded.starfish_rbc_dag_autonomous_clock); + assert_eq!( + decoded.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() + ); + } + + #[test] + fn autonomous_clock_configuration_roundtrips() { + let parameters = NodeParameters { + starfish_rbc_dag_shadow: true, + starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, + ..NodeParameters::default() + }; + + let yaml = serde_yaml::to_string(¶meters).unwrap(); + let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); + assert!(decoded.starfish_rbc_dag_shadow); + assert!(decoded.starfish_rbc_dag_autonomous_clock); + assert_eq!(decoded.starfish_rbc_dag_heartbeat_interval_ms, 125); + } + + #[test] + fn autonomous_clock_uses_a_distinct_wal() { + let private_config = + NodePrivateConfig::new_for_benchmarks(Path::new("benchmark"), 1).remove(0); + + assert_ne!( + private_config.starfish_rbc_dag_shadow_wal(), + private_config.starfish_rbc_dag_autonomous_clock_wal() + ); + assert_eq!( + private_config.starfish_rbc_dag_autonomous_clock_wal(), + Path::new("benchmark") + .join("storage-0") + .join("starfish-rbc-dag-autonomous-clock-v1.wal") + ); } } diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 83487543..cbb56c7a 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -41,6 +41,16 @@ pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR: i64 = 4; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; +/// Local-benchmark guards for the non-authoritative autonomous carrier +/// clock. The round-skew limit matches the executable model's bounded future +/// buffer. A healthy clock can transiently retain phase work, but its carrier +/// capacity exceeds the two RBC statements generated per admitted value; a +/// sixteen-committee backlog therefore leaves generous scheduling headroom +/// while still detecting an actor that is no longer draining work. +pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG: i64 = 4; +pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR: i64 = 16; +pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR: i64 = 2; + #[derive(Clone)] pub struct Metrics { pub benchmark_duration: IntCounter, @@ -154,6 +164,12 @@ pub struct Metrics { pub starfish_rbc_dag_shadow_unpaired_shadow: IntGauge, pub starfish_rbc_dag_shadow_unpaired_max_round_lag: IntGauge, pub starfish_rbc_dag_shadow_comparison_valid: IntGauge, + pub starfish_rbc_dag_shadow_clock_valid: IntGauge, + pub starfish_rbc_dag_shadow_carrier_round: IntGauge, + pub starfish_rbc_dag_shadow_phase_backlog: IntGauge, + pub starfish_rbc_dag_shadow_admitted_authors: IntGauge, + pub starfish_rbc_dag_shadow_admitted_stake: IntGauge, + pub starfish_rbc_dag_shadow_buffered_authenticated: IntGauge, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -217,6 +233,199 @@ pub struct MetricReporter { pub global_in_memory_blocks_bytes: IntGauge, } +/// Per-validator counters captured after the autonomous shadow becomes ready +/// and immediately before the measured local-benchmark interval begins. +#[derive(Clone, Copy, Debug, Default)] +pub struct AutonomousClockBenchmarkBaseline { + accepted_heartbeats: u64, + delivered_carriers: u64, + wal_batches: u64, + wal_records: u64, + carrier_round: i64, +} + +#[derive(Debug, Eq, PartialEq)] +struct AutonomousClockBenchmarkSummary { + valid_nodes: usize, + progress_nodes: usize, + bounded_nodes: usize, + accepted_heartbeats: u64, + delivered_carriers: u64, + wal_batches: u64, + wal_records: u64, + pending_recovery: i64, + minimum_round: i64, + maximum_round: i64, + maximum_phase_backlog: i64, + maximum_admitted_authors: i64, + maximum_admitted_stake: i64, + maximum_buffered_authenticated: i64, + maximum_phase_backlog_bound: i64, + maximum_buffered_authenticated_bound: i64, + verdict_valid: bool, +} + +fn summarize_autonomous_clock_benchmark( + metrics: &[Arc], + committee_size: usize, + baselines: Option<&[AutonomousClockBenchmarkBaseline]>, +) -> AutonomousClockBenchmarkSummary { + let committee_size = i64::try_from(committee_size).unwrap_or(i64::MAX); + let maximum_phase_backlog_bound = + STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR.saturating_mul(committee_size); + let maximum_buffered_authenticated_bound = + STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR.saturating_mul(committee_size); + + let valid_nodes = metrics + .iter() + .filter(|metrics| metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1) + .count(); + let progress_nodes = metrics + .iter() + .enumerate() + .filter(|(index, metrics)| { + let baseline = baselines + .and_then(|baselines| baselines.get(*index)) + .copied() + .unwrap_or_default(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .get() + > baseline.accepted_heartbeats + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get() + > baseline.delivered_carriers + && metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .get() + > baseline.wal_batches + && metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .get() + > baseline.wal_records + && metrics.starfish_rbc_dag_shadow_carrier_round.get() > baseline.carrier_round + }) + .count(); + let bounded_nodes = metrics + .iter() + .filter(|metrics| { + let phase_backlog = metrics.starfish_rbc_dag_shadow_phase_backlog.get(); + let admitted_authors = metrics.starfish_rbc_dag_shadow_admitted_authors.get(); + let admitted_stake = metrics.starfish_rbc_dag_shadow_admitted_stake.get(); + let buffered = metrics.starfish_rbc_dag_shadow_buffered_authenticated.get(); + phase_backlog >= 0 + && phase_backlog <= maximum_phase_backlog_bound + && admitted_authors >= 0 + && admitted_authors <= committee_size + && admitted_stake >= 0 + && buffered >= 0 + && buffered <= maximum_buffered_authenticated_bound + && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 + }) + .count(); + + let accepted_heartbeats = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .get() + }) + .sum(); + let delivered_carriers = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get() + }) + .sum(); + let wal_batches = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .get() + }) + .sum(); + let wal_records = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .get() + }) + .sum(); + let pending_recovery = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_pending_recovery.get()) + .sum(); + let minimum_round = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) + .min() + .unwrap_or_default(); + let maximum_round = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) + .max() + .unwrap_or_default(); + let maximum_phase_backlog = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_phase_backlog.get()) + .max() + .unwrap_or_default(); + let maximum_admitted_authors = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_authors.get()) + .max() + .unwrap_or_default(); + let maximum_admitted_stake = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_stake.get()) + .max() + .unwrap_or_default(); + let maximum_buffered_authenticated = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_shadow_buffered_authenticated.get()) + .max() + .unwrap_or_default(); + let round_lag = maximum_round.saturating_sub(minimum_round); + let every_node_valid = valid_nodes == metrics.len(); + let every_node_progressed = progress_nodes == metrics.len(); + let every_node_bounded = bounded_nodes == metrics.len(); + let verdict_valid = !metrics.is_empty() + && every_node_valid + && every_node_progressed + && every_node_bounded + && round_lag <= STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG; + + AutonomousClockBenchmarkSummary { + valid_nodes, + progress_nodes, + bounded_nodes, + accepted_heartbeats, + delivered_carriers, + wal_batches, + wal_records, + pending_recovery, + minimum_round, + maximum_round, + maximum_phase_backlog, + maximum_admitted_authors, + maximum_admitted_stake, + maximum_buffered_authenticated, + maximum_phase_backlog_bound, + maximum_buffered_authenticated_bound, + verdict_valid, + } +} + pub struct HistogramReporter { pub histogram: PreciseHistogram, gauge: IntGaugeVec, @@ -228,6 +437,22 @@ pub struct VecHistogramReporter { } impl Metrics { + pub fn autonomous_clock_benchmark_baseline(&self) -> AutonomousClockBenchmarkBaseline { + AutonomousClockBenchmarkBaseline { + accepted_heartbeats: self + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .get(), + delivered_carriers: self + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get(), + wal_batches: self.starfish_rbc_dag_shadow_wal_durable_batches_total.get(), + wal_records: self.starfish_rbc_dag_shadow_wal_durable_records_total.get(), + carrier_round: self.starfish_rbc_dag_shadow_carrier_round.get(), + } + } + pub fn new( registry: &Registry, committee: Option<&Committee>, @@ -606,6 +831,42 @@ impl Metrics { registry, ) .unwrap(), + starfish_rbc_dag_shadow_clock_valid: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_clock_valid", + "State of the non-authoritative autonomous carrier clock (1 valid, 0 disabled/invalid, -1 starting)", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_carrier_round: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_carrier_round", + "Currently open sequential Starfish-RBC-DAG shadow carrier round", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_phase_backlog: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_phase_backlog", + "Pending embedded ECHO/READY statements in the autonomous carrier actor", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_admitted_authors: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_admitted_authors", + "Distinct authors admitted in the currently open carrier round", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_admitted_stake: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_admitted_stake", + "Stake admitted in the currently open carrier round", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_buffered_authenticated: register_int_gauge_with_registry!( + "starfish_rbc_dag_shadow_buffered_authenticated", + "Authenticated future carrier slots buffered outside the current admission window", + registry, + ) + .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1064,6 +1325,8 @@ impl Metrics { duration_secs: u64, committee_size: usize, starfish_rbc_dag_shadow_expected: bool, + starfish_rbc_dag_autonomous_clock_expected: bool, + autonomous_clock_baselines: Option>, ) { let num_validators = metrics.len() as u64; @@ -1204,6 +1467,8 @@ impl Metrics { "rbc_dag_shadow_carrier", "rbc_dag_shadow_carrier_request", "rbc_dag_shadow_carrier_response", + "rbc_dag_shadow_carrier_sync_request", + "rbc_dag_shadow_carrier_sync_response", ]; let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() @@ -1259,7 +1524,64 @@ impl Metrics { }; table.add_row(row![b->"Bandwidth efficiency:", format!("{:.2}", bandwidth_efficiency)]); - if starfish_rbc_dag_shadow_expected { + if starfish_rbc_dag_autonomous_clock_expected { + let summary = summarize_autonomous_clock_benchmark( + &metrics, + committee_size, + autonomous_clock_baselines.as_deref(), + ); + let round_lag = summary.maximum_round.saturating_sub(summary.minimum_round); + + table.add_row(row![bH2->""]); + table.add_row(row![bH2->"RBC-DAG Autonomous Clock Verification"]); + table.add_row(row![ + b->"Clock verdict:", + if summary.verdict_valid { + "VALID".to_owned() + } else { + "INVALID — DISCARD THIS AUTONOMOUS-CLOCK RUN".to_owned() + } + ]); + table.add_row(row![ + b->"Valid/progress/bounded validators:", + format!( + "{}/{}, {}/{}, {}/{}", + summary.valid_nodes, + metrics.len(), + summary.progress_nodes, + metrics.len(), + summary.bounded_nodes, + metrics.len(), + ) + ]); + table.add_row(row![ + b->"Durable clock progress:", + format!( + "heartbeats={}, RBC deliveries={}, WAL batches={}, records={}, open rounds={}..{}", + summary.accepted_heartbeats, + summary.delivered_carriers, + summary.wal_batches, + summary.wal_records, + summary.minimum_round, + summary.maximum_round, + ) + ]); + table.add_row(row![ + b->"Bounded live state:", + format!( + "round skew={round_lag}/{}, max phase backlog={}/{}, admitted authors={}/{}, stake={}, max buffered={}/{}, pending recovery={}", + STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, + summary.maximum_phase_backlog, + summary.maximum_phase_backlog_bound, + summary.maximum_admitted_authors, + committee_size, + summary.maximum_admitted_stake, + summary.maximum_buffered_authenticated, + summary.maximum_buffered_authenticated_bound, + summary.pending_recovery, + ) + ]); + } else if starfish_rbc_dag_shadow_expected { let valid_nodes = metrics .iter() .filter(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) @@ -1740,6 +2062,14 @@ mod tests { .starfish_rbc_dag_shadow_unpaired_max_round_lag .set(8); metrics.starfish_rbc_dag_shadow_comparison_valid.set(1); + metrics.starfish_rbc_dag_shadow_clock_valid.set(1); + metrics.starfish_rbc_dag_shadow_carrier_round.set(9); + metrics.starfish_rbc_dag_shadow_phase_backlog.set(10); + metrics.starfish_rbc_dag_shadow_admitted_authors.set(3); + metrics.starfish_rbc_dag_shadow_admitted_stake.set(3); + metrics + .starfish_rbc_dag_shadow_buffered_authenticated + .set(2); let gathered = registry.gather(); for name in [ @@ -1754,6 +2084,12 @@ mod tests { "starfish_rbc_dag_shadow_unpaired_shadow", "starfish_rbc_dag_shadow_unpaired_max_round_lag", "starfish_rbc_dag_shadow_comparison_valid", + "starfish_rbc_dag_shadow_clock_valid", + "starfish_rbc_dag_shadow_carrier_round", + "starfish_rbc_dag_shadow_phase_backlog", + "starfish_rbc_dag_shadow_admitted_authors", + "starfish_rbc_dag_shadow_admitted_stake", + "starfish_rbc_dag_shadow_buffered_authenticated", ] { assert!( gathered.iter().any(|family| family.get_name() == name), @@ -1761,4 +2097,123 @@ mod tests { ); } } + + fn autonomous_clock_metrics( + round: i64, + phase_backlog: i64, + buffered_authenticated: i64, + ) -> Arc { + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new(®istry, None, None, None); + metrics.starfish_rbc_dag_shadow_clock_valid.set(1); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .inc_by(2); + metrics.starfish_rbc_dag_shadow_carrier_round.set(round); + metrics + .starfish_rbc_dag_shadow_phase_backlog + .set(phase_backlog); + metrics.starfish_rbc_dag_shadow_admitted_authors.set(2); + metrics.starfish_rbc_dag_shadow_admitted_stake.set(2); + metrics + .starfish_rbc_dag_shadow_buffered_authenticated + .set(buffered_authenticated); + metrics + } + + #[test] + fn autonomous_clock_summary_requires_every_node_to_make_bounded_durable_progress() { + let metrics = vec![ + autonomous_clock_metrics(8, 3, 1), + autonomous_clock_metrics(9, 4, 2), + autonomous_clock_metrics(10, 5, 0), + autonomous_clock_metrics(11, 6, 1), + ]; + + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); + + assert!(summary.verdict_valid); + assert_eq!(summary.valid_nodes, 4); + assert_eq!(summary.progress_nodes, 4); + assert_eq!(summary.bounded_nodes, 4); + assert_eq!(summary.accepted_heartbeats, 4); + assert_eq!(summary.delivered_carriers, 4); + assert_eq!(summary.wal_batches, 4); + assert_eq!(summary.wal_records, 8); + assert_eq!(summary.minimum_round, 8); + assert_eq!(summary.maximum_round, 11); + } + + #[test] + fn autonomous_clock_summary_requires_progress_after_the_benchmark_baseline() { + let metrics = vec![ + autonomous_clock_metrics(8, 0, 0), + autonomous_clock_metrics(8, 0, 0), + ]; + let baselines = metrics + .iter() + .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) + .collect::>(); + + assert!(!summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); + + for metrics in &metrics { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .inc(); + metrics.starfish_rbc_dag_shadow_carrier_round.inc(); + } + + assert!(summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); + } + + #[test] + fn autonomous_clock_summary_rejects_invalid_progress_and_unbounded_state() { + let no_progress = autonomous_clock_metrics(1, 0, 0); + no_progress + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .reset(); + let invalid_clock = autonomous_clock_metrics(12, 0, 0); + invalid_clock.starfish_rbc_dag_shadow_clock_valid.set(0); + let unbounded = autonomous_clock_metrics( + 20, + STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR * 4 + 1, + STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR * 4 + 1, + ); + let metrics = vec![no_progress, invalid_clock, unbounded]; + + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); + + assert!(!summary.verdict_valid); + assert_eq!(summary.valid_nodes, 2); + assert_eq!(summary.progress_nodes, 2); + assert_eq!(summary.bounded_nodes, 2); + assert!( + summary.maximum_round - summary.minimum_round + > STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG + ); + } } diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 819a586d..e1089edd 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -35,7 +35,7 @@ use crate::{ }, core::Core, core_thread::CoreThreadDispatcher, - crypto::{BlsSigner, MacKey}, + crypto::{Blake3Hasher, BlsSigner, MacKey}, dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, metrics::{Metrics, UtilizationTimerVecExt}, @@ -52,6 +52,7 @@ use crate::{ }, starfish_rbc_dag_shadow_service::{ ShadowServiceErrorV1, ShadowServiceEventV1, StarfishRbcDagShadowServiceHandleV1, + start_starfish_rbc_dag_autonomous_clock_service_v1, start_starfish_rbc_dag_shadow_service_v1, }, starfish_rbc_service::{ @@ -70,6 +71,8 @@ const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); const STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); +const STARFISH_RBC_DAG_AUTONOMOUS_INSTANCE_CONTEXT: &str = + "STARFISH_RBC_DAG_AUTONOMOUS_CLOCK_V1_PROTOCOL_INSTANCE"; /// Recover the exact locally selected Starfish-RBC chain so the persisted /// non-authoritative shadow can reconcile a WAL that ended before the direct @@ -139,13 +142,40 @@ fn recovered_local_rbc_headers( Ok(reversed) } -fn shadow_transport_error_invalidates_comparison(error: &ShadowServiceErrorV1) -> bool { +fn shadow_transport_error_invalidates_run(error: &ShadowServiceErrorV1) -> bool { matches!( error, ShadowServiceErrorV1::Overloaded { .. } | ShadowServiceErrorV1::Stopped ) } +fn invalidate_shadow_run(metrics: &Metrics) { + // Exactly one verdict is active for a configured shadow mode, but setting + // both to zero makes every transport/startup failure fail closed without + // duplicating mode knowledge throughout the network plumbing. + metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + metrics.starfish_rbc_dag_shadow_clock_valid.set(0); +} + +fn rbc_dag_shadow_protocol_instance( + direct_instance: [u8; 32], + autonomous_clock: bool, +) -> RbcDagProtocolInstanceId { + let bytes = if autonomous_clock { + // Autonomous heartbeat carriers intentionally do not authenticate in + // the same namespace as milestone-three's direct-header mirror. This + // makes a heterogeneous deployment fail closed at the shadow boundary + // instead of cross-admitting application and control carriers. + let mut hasher = Blake3Hasher::new_derive_key(STARFISH_RBC_DAG_AUTONOMOUS_INSTANCE_CONTEXT); + hasher.update(&direct_instance); + *hasher.finalize().as_bytes() + } else { + direct_instance + }; + RbcDagProtocolInstanceId::new(bytes) + .expect("a configured direct RBC instance and its derived namespace are nonzero") +} + /// Enforce the MAC experiment's transport contract before cryptographic /// verification: /// @@ -1075,8 +1105,8 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier(self.peer_id, envelope) { - if shadow_transport_error_invalidates_comparison(&error) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); } tracing::warn!("Failed to forward RBC-DAG shadow carrier: {error}"); } @@ -1085,8 +1115,8 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier_request(self.peer_id, reference) { - if shadow_transport_error_invalidates_comparison(&error) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); } tracing::warn!("Failed to forward RBC-DAG shadow request: {error}"); } @@ -1095,13 +1125,37 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier_response(self.peer_id, response) { - if shadow_transport_error_invalidates_comparison(&error) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); } tracing::warn!("Failed to forward RBC-DAG shadow response: {error}"); } } } + NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.carrier_sync_request(self.peer_id, request) { + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); + } + tracing::warn!( + "Failed to forward RBC-DAG shadow carrier sync request: {error}" + ); + } + } + } + NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.carrier_sync_response(self.peer_id, response) { + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); + } + tracing::warn!( + "Failed to forward RBC-DAG shadow carrier sync response: {error}" + ); + } + } + } } true } @@ -1752,7 +1806,13 @@ impl NetworkSyncer let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); - let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { + let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow + && node_parameters.starfish_rbc_dag_autonomous_clock + { + // Autonomous carrier rounds are independent of direct consensus + // rounds and recover entirely from their distinct WAL. + Some(Vec::new()) + } else if node_parameters.starfish_rbc_dag_shadow { match recovered_local_rbc_headers(&core) { Ok(headers) => Some(headers), Err(error) => { @@ -1837,8 +1897,10 @@ impl NetworkSyncer let protocol_instance_bytes = node_parameters .starfish_rbc_protocol_instance .expect("validated shadow configuration must share the direct RBC instance"); - let protocol_instance = RbcDagProtocolInstanceId::new(protocol_instance_bytes) - .expect("validated direct RBC instance must be nonzero"); + let protocol_instance = rbc_dag_shadow_protocol_instance( + protocol_instance_bytes, + node_parameters.starfish_rbc_dag_autonomous_clock, + ); let committee_context = RbcDagCommitteeContextV1::new(committee.clone()) .expect("validated committee must initialize the RBC-DAG shadow"); let context = RbcDagContextV1::new_with_committee( @@ -1861,19 +1923,40 @@ impl NetworkSyncer } }; // -1 means the background WAL replay has not completed yet; - // Ready moves this to 1 unless work was already shed (0). - metrics.starfish_rbc_dag_shadow_comparison_valid.set(-1); - match start_starfish_rbc_dag_shadow_service_v1( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - recovered_local_headers, - ) { + // Ready moves the active observational mode to 1 unless work + // was already shed (0). The inactive verdict remains zero. + if node_parameters.starfish_rbc_dag_autonomous_clock { + metrics.starfish_rbc_dag_shadow_clock_valid.set(-1); + metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + } else { + metrics.starfish_rbc_dag_shadow_comparison_valid.set(-1); + metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + } + let started = if node_parameters.starfish_rbc_dag_autonomous_clock { + start_starfish_rbc_dag_autonomous_clock_service_v1( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + Duration::from_millis( + node_parameters.starfish_rbc_dag_heartbeat_interval_ms, + ), + ) + } else { + start_starfish_rbc_dag_shadow_service_v1( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + recovered_local_headers, + ) + }; + match started { Ok((service, events, task)) => (Some(service), Some(events), Some(task)), Err(error) => { - metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + invalidate_shadow_run(&metrics); tracing::error!( "Disabling non-authoritative Starfish-RBC-DAG shadow: {error}" ); @@ -2076,10 +2159,8 @@ impl NetworkSyncer canonical.transactions_commitment(), ); if let Err(error) = shadow.direct_delivered(identity) { - if shadow_transport_error_invalidates_comparison(&error) { - rbc_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&rbc_metrics); } tracing::warn!( "Failed to notify RBC-DAG shadow of direct delivery: {error}" @@ -2128,6 +2209,7 @@ impl NetworkSyncer shadow_metrics .starfish_rbc_dag_shadow_comparison_valid .set(0); + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } Err(mpsc::error::TrySendError::Closed(_)) => { shadow_metrics @@ -2137,6 +2219,7 @@ impl NetworkSyncer shadow_metrics .starfish_rbc_dag_shadow_comparison_valid .set(0); + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } } } else { @@ -2208,17 +2291,39 @@ impl NetworkSyncer .starfish_rbc_dag_shadow_wal_durable_records_total .inc_by(records); } - ShadowServiceEventV1::Ready => { - if shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .get() - != 0 - { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(1); + ShadowServiceEventV1::Ready { autonomous_clock } => { + let verdict = if autonomous_clock { + &shadow_metrics.starfish_rbc_dag_shadow_clock_valid + } else { + &shadow_metrics.starfish_rbc_dag_shadow_comparison_valid + }; + if verdict.get() != 0 { + verdict.set(1); } } + ShadowServiceEventV1::ClockState { + open_round, + phase_backlog, + admitted_authors, + admitted_stake, + buffered_authenticated, + } => { + shadow_metrics + .starfish_rbc_dag_shadow_carrier_round + .set(i64::from(open_round)); + shadow_metrics + .starfish_rbc_dag_shadow_phase_backlog + .set(i64::try_from(phase_backlog).unwrap_or(i64::MAX)); + shadow_metrics + .starfish_rbc_dag_shadow_admitted_authors + .set(i64::try_from(admitted_authors).unwrap_or(i64::MAX)); + shadow_metrics + .starfish_rbc_dag_shadow_admitted_stake + .set(i64::try_from(admitted_stake).unwrap_or(i64::MAX)); + shadow_metrics + .starfish_rbc_dag_shadow_buffered_authenticated + .set(i64::try_from(buffered_authenticated).unwrap_or(i64::MAX)); + } ShadowServiceEventV1::Recovered { batches, discarded_tail_bytes, @@ -2240,6 +2345,7 @@ impl NetworkSyncer shadow_metrics .starfish_rbc_dag_shadow_comparison_valid .set(0); + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } tracing::warn!( "Rejected non-authoritative RBC-DAG shadow input from {:?}: {}", @@ -2804,10 +2910,8 @@ impl NetworkSyncer } if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.peer_connected(peer_id) { - if shadow_transport_error_invalidates_comparison(&error) { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&shadow_metrics); } tracing::warn!( "Failed to notify RBC-DAG shadow that authority {} connected: {}", @@ -2865,10 +2969,8 @@ impl NetworkSyncer } if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.peer_disconnected(peer_id) { - if shadow_transport_error_invalidates_comparison(&error) { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&shadow_metrics); } tracing::warn!( "Failed to notify RBC-DAG shadow that authority {} disconnected: {}", @@ -3583,4 +3685,19 @@ mod tests { assert_eq!(unique.len(), selected.len()); assert!(selected.iter().all(|peer| candidates.contains(peer))); } + + #[test] + fn autonomous_carriers_use_a_distinct_authentication_namespace() { + let direct = [0x5A; 32]; + let mirror = rbc_dag_shadow_protocol_instance(direct, false); + let autonomous = rbc_dag_shadow_protocol_instance(direct, true); + + assert_eq!(mirror.as_bytes(), &direct); + assert_ne!(autonomous, mirror); + assert_eq!( + autonomous, + rbc_dag_shadow_protocol_instance(direct, true), + "derived autonomous namespace must be deterministic across nodes" + ); + } } diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 6531bd77..82805482 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -101,6 +101,26 @@ pub struct RbcDagShadowCarrierResponse { pub canonical_carrier: Vec, } +/// Request one exact carrier-clock slot from a peer. Keeping synchronization +/// slot-addressed prevents an untrusted peer from choosing an unbounded range +/// of history to return. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcDagShadowCarrierSyncRequest { + pub author: AuthorityIndex, + pub round: RoundNumber, +} + +/// Full response for one exact carrier-clock slot. The receiver validates that +/// the canonical carrier has the requested author and round, and authenticates +/// the sidecar before admitting it. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcDagShadowCarrierSyncResponse { + pub author: AuthorityIndex, + pub round: RoundNumber, + pub canonical_carrier: Vec, + pub authentication_sidecar: Vec, +} + /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -216,6 +236,12 @@ pub enum NetworkMessage { RbcDagShadowCarrierRequest(BlockReference), /// Return content only; the receiver recomputes and checks the reference. RbcDagShadowCarrierResponse(RbcDagShadowCarrierResponse), + /// Starfish-RBC-DAG milestone-four synchronization for one exact + /// `(author, round)` carrier-clock slot. + RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest), + /// Full canonical carrier and authentication sidecar for an exact + /// carrier-clock slot. Receivers validate the duplicated slot identity. + RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse), } impl NetworkMessage { @@ -246,6 +272,8 @@ impl NetworkMessage { Self::RbcDagShadowCarrier(_) => "rbc_dag_shadow_carrier", Self::RbcDagShadowCarrierRequest(_) => "rbc_dag_shadow_carrier_request", Self::RbcDagShadowCarrierResponse(_) => "rbc_dag_shadow_carrier_response", + Self::RbcDagShadowCarrierSyncRequest(_) => "rbc_dag_shadow_carrier_sync_request", + Self::RbcDagShadowCarrierSyncResponse(_) => "rbc_dag_shadow_carrier_sync_response", } } } @@ -1152,6 +1180,18 @@ mod tests { reference: block_ref, canonical_carrier: vec![0xA6, 0xA7], }); + let sync_request = + NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { + author: 2, + round: 23, + }); + let sync_response = + NetworkMessage::RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse { + author: 2, + round: 23, + canonical_carrier: vec![0xA8, 0xA9], + authentication_sidecar: vec![0xAA, 0xAB], + }); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), @@ -1161,6 +1201,8 @@ mod tests { (shadow, 15, "rbc_dag_shadow_carrier"), (shadow_request, 16, "rbc_dag_shadow_carrier_request"), (shadow_response, 17, "rbc_dag_shadow_carrier_response"), + (sync_request, 18, "rbc_dag_shadow_carrier_sync_request"), + (sync_response, 19, "rbc_dag_shadow_carrier_sync_response"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); @@ -1171,6 +1213,37 @@ mod tests { } } + #[test] + fn rbc_dag_shadow_carrier_sync_payloads_roundtrip_exactly() { + let request = RbcDagShadowCarrierSyncRequest { + author: 3, + round: 41, + }; + let encoded = + bincode::serialize(&NetworkMessage::RbcDagShadowCarrierSyncRequest(request)).unwrap(); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + assert!(matches!( + decoded, + NetworkMessage::RbcDagShadowCarrierSyncRequest(decoded) if decoded == request + )); + + let response = RbcDagShadowCarrierSyncResponse { + author: 3, + round: 41, + canonical_carrier: vec![0xC1, 0xC2, 0xC3], + authentication_sidecar: vec![0xD1, 0xD2], + }; + let encoded = bincode::serialize(&NetworkMessage::RbcDagShadowCarrierSyncResponse( + response.clone(), + )) + .unwrap(); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + assert!(matches!( + decoded, + NetworkMessage::RbcDagShadowCarrierSyncResponse(decoded) if decoded == response + )); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn scoped_connection_tasks_allow_immediate_same_port_rebind() { // Active sockets bind to listener_port * 10. Keep those derived ports diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index ba10e87f..8d62a392 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -34,7 +34,7 @@ use crate::{ }, types::{ AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, - RoundNumber, TimestampNs, + RoundNumber, Stake, TimestampNs, }, }; @@ -551,6 +551,49 @@ impl StarfishRbcDagShadowV1 { self.model.can_create_carrier() } + pub(crate) fn pending_phase_backlog_len(&self) -> usize { + self.model.pending_phase_backlog_len() + } + + pub(crate) fn admitted_reference( + &self, + authority: AuthorityIndex, + round: RoundNumber, + ) -> Option { + self.model.admitted_reference(authority, round) + } + + pub(crate) fn current_round_admitted_author_count(&self) -> usize { + let round = self.local_carrier_round(); + self.committee + .committee() + .authorities() + .filter(|authority| self.model.admitted_reference(*authority, round).is_some()) + .count() + } + + pub(crate) fn current_round_admitted_stake(&self) -> Stake { + let round = self.local_carrier_round(); + self.committee + .committee() + .authorities() + .filter(|authority| self.model.admitted_reference(*authority, round).is_some()) + .filter_map(|authority| self.committee.committee().get_stake(authority)) + .fold(0, Stake::saturating_add) + } + + /// Authenticated slots retained beyond the model's current admission + /// window. These are bounded by the reducer's future-carrier window and + /// become admitted only through sequential clock advancement. + pub(crate) fn buffered_authenticated_carrier_count(&self) -> usize { + self.authenticated_slots + .iter() + .filter(|((authority, round), reference)| { + self.model.admitted_reference(*authority, *round) != Some(**reference) + }) + .count() + } + pub(crate) fn wal_counts(&self) -> (u64, u64) { (self.wal.batch_count(), self.wal.record_count()) } @@ -608,6 +651,17 @@ impl StarfishRbcDagShadowV1 { Ok((envelope, effects)) } + /// Create the currently open autonomous control slot with no application + /// payload. The caller supplies only a timestamp: the round is derived + /// from the durable reducer and the empty commitment is canonical. + pub(crate) fn create_local_control_heartbeat( + &mut self, + creation_time_ns: TimestampNs, + ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { + let round = self.model.local_carrier_round(); + self.create_local_carrier(round, TransactionsCommitment::default(), creation_time_ns) + } + /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -784,6 +838,37 @@ impl StarfishRbcDagShadowV1 { .map(<[u8]>::to_vec) } + /// Decode canonical carrier bytes without mutating the reducer. Sync + /// clients use this to bind a response to the requested author and round + /// before passing it through normal authenticated ingress. + pub(crate) fn candidate_slot( + &self, + canonical_carrier_wire: &[u8], + ) -> Result<(AuthorityIndex, RoundNumber, BlockReference), ShadowErrorV1> { + let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; + Ok(( + candidate.header().author(), + candidate.header().carrier_round(), + candidate.reference(), + )) + } + + /// Return the exact durably exposed local envelope for one carrier round. + /// Missing, incomplete, or unexposed slots are never served. + pub(crate) fn local_outbound_envelope( + &self, + round: RoundNumber, + ) -> Option { + let snapshot = self.journal.snapshot(); + let reference = snapshot.own_carrier(round)?; + let outbound = snapshot.outbound(reference)?; + outbound.exposed().then(|| ShadowOutboundEnvelopeV1 { + reference: outbound.reference(), + canonical_carrier_wire: outbound.canonical_carrier_wire().to_vec(), + authentication_sidecar: outbound.authentication_sidecar().to_vec(), + }) + } + /// Return every exposed local carrier in deterministic reference order. /// The same full sidecar is returned for every peer. pub(crate) fn retransmissions(&self) -> Vec { @@ -805,7 +890,7 @@ impl StarfishRbcDagShadowV1 { /// payload and creation timestamp before accepting new observations. pub(crate) fn local_outbound_metadata( &self, - ) -> Result, ShadowErrorV1> { + ) -> Result, ShadowErrorV1> { self.journal .snapshot() .retransmissions() @@ -820,6 +905,8 @@ impl StarfishRbcDagShadowV1 { candidate.header().carrier_round(), candidate.header().transactions_commitment(), candidate.header().creation_time_ns(), + candidate.header().data_acknowledgments().is_empty() + && candidate.header().consensus_vertex().is_none(), )) }) .collect() @@ -2053,6 +2140,139 @@ mod tests { ); } + #[test] + fn autonomous_control_heartbeat_derives_the_open_round_and_is_durably_addressable() { + let mut network = TestNetwork::new(); + let node = &mut network.nodes[0]; + assert_eq!(node.local_carrier_round(), 1); + assert_eq!(node.current_round_admitted_author_count(), 0); + assert_eq!(node.current_round_admitted_stake(), 0); + assert_eq!(node.pending_phase_backlog_len(), 0); + assert_eq!(node.buffered_authenticated_carrier_count(), 0); + assert_eq!(node.local_outbound_envelope(1), None); + + let before = node.wal_counts(); + let (heartbeat, effects) = node.create_local_control_heartbeat(123).unwrap(); + assert!(effects.is_empty()); + assert_eq!(node.wal_counts().0, before.0 + 1); + let candidate = decode_candidate( + heartbeat.canonical_carrier_wire(), + &network.committee, + Some(heartbeat.reference()), + ) + .unwrap(); + assert_eq!(candidate.header().author(), 0); + assert_eq!(candidate.header().carrier_round(), 1); + assert_eq!( + candidate.header().transactions_commitment(), + TransactionsCommitment::default() + ); + assert_eq!(candidate.header().creation_time_ns(), 123); + assert!(candidate.header().data_acknowledgments().is_empty()); + assert!(candidate.header().phase_batch().is_empty()); + assert!(candidate.header().consensus_vertex().is_none()); + assert_eq!(node.local_outbound_envelope(1), Some(heartbeat.clone())); + assert_eq!(node.local_outbound_envelope(2), None); + assert_eq!( + node.candidate_slot(heartbeat.canonical_carrier_wire()) + .unwrap(), + (0, 1, heartbeat.reference()) + ); + assert_eq!(node.admitted_reference(0, 1), Some(heartbeat.reference())); + assert_eq!(node.current_round_admitted_author_count(), 1); + assert_eq!(node.current_round_admitted_stake(), 1); + assert_eq!(node.pending_phase_backlog_len(), 1); + assert_eq!(node.buffered_authenticated_carrier_count(), 0); + + let durable_counts = node.wal_counts(); + assert!(matches!( + node.create_local_control_heartbeat(124), + Err(ShadowErrorV1::Model(ModelError::LocalCarrierAlreadyFixed( + 1 + ))) + )); + assert_eq!(node.wal_counts(), durable_counts); + + let mut trailing = heartbeat.canonical_carrier_wire().to_vec(); + trailing.push(0); + assert!(node.candidate_slot(&trailing).is_err()); + } + + #[test] + fn autonomous_control_heartbeat_advances_sequentially_and_reopens_exact_bytes() { + let mut network = TestNetwork::new(); + let first = network.nodes[0] + .create_local_control_heartbeat(1_000) + .unwrap() + .0; + for author in [1, 2] { + let candidate = round_one_candidate(author, &network.committee, 0x70 + author as u8); + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: author, + keys: &network.keyrings[author as usize], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &candidate.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + author, + ) + .unwrap(); + } + assert_eq!(network.nodes[0].local_carrier_round(), 2); + assert!(network.nodes[0].can_create_carrier()); + assert_eq!(network.nodes[0].current_round_admitted_author_count(), 0); + + let second = network.nodes[0] + .create_local_control_heartbeat(2_000) + .unwrap() + .0; + let second_candidate = decode_candidate( + second.canonical_carrier_wire(), + &network.committee, + Some(second.reference()), + ) + .unwrap(); + assert_eq!(second_candidate.header().carrier_round(), 2); + assert_eq!(second_candidate.header().own_prev(), first.reference()); + assert_eq!( + second_candidate.header().transactions_commitment(), + TransactionsCommitment::default() + ); + assert_eq!( + network.nodes[0].local_outbound_envelope(1), + Some(first.clone()) + ); + assert_eq!( + network.nodes[0].local_outbound_envelope(2), + Some(second.clone()) + ); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let (restarted, report) = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap(); + assert!(report.replayed_batches() >= 4); + assert_eq!(restarted.local_carrier_round(), 2); + assert!(!restarted.can_create_carrier()); + assert_eq!(restarted.local_outbound_envelope(1), Some(first)); + assert_eq!(restarted.local_outbound_envelope(2), Some(second)); + } + #[test] fn authenticated_replays_and_slot_conflicts_do_not_grow_durable_state() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 49879cc2..27787191 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -8,8 +8,11 @@ use std::{ error::Error, fmt, path::Path, - sync::Arc, - time::{Duration, Instant}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use parking_lot::Mutex; @@ -24,7 +27,10 @@ use tokio::{ use crate::{ crypto::{MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE}, - network::{NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse}, + network::{ + NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse, + RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, + }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, @@ -38,13 +44,14 @@ use crate::{ types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, }; -// A shadow run must absorb one complete committee fan-in plus a small reserve -// for the local carrier and control notifications before its single fsync -// owner can drain. At the four-MiB carrier cap, allowing at most 64 queued -// inputs also caps carrier payload retention at 256 MiB (plus bounded -// sidecars and allocator overhead). Larger committees are rejected for this -// benchmark prototype instead of silently under-sizing the queue and -// reporting incomparable results. +// A mirror run must absorb one complete committee fan-in plus a small reserve; +// autonomous repair additionally budgets a simultaneous request and response +// per peer. At the four-MiB carrier cap, allowing at most 64 queued inputs also +// caps carrier payload retention at 256 MiB (plus bounded sidecars and +// allocator overhead). This permits 60 mirror validators or 20 autonomous +// validators. Larger committees are rejected for this benchmark prototype +// instead of silently under-sizing the queue and reporting incomparable +// results. // Use the full bounded allowance even for a small committee. A single fan-in // reserve is insufficient when several round bursts arrive while the actor is // synchronously making the previous transition durable. @@ -52,7 +59,42 @@ const SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1: usize = 64; const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 64; const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; +const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); +const SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1: Duration = Duration::from_millis(100); +const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_millis(500); + +/// Runtime role of the persisted carrier actor. +/// +/// Mirror mode preserves milestone three's one-to-one comparison against +/// direct Starfish-RBC headers. Autonomous mode opens an independent, +/// heartbeat-only carrier clock. It remains observational: neither mode can +/// call the core dispatcher or mutate authoritative consensus state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ShadowServiceModeV1 { + DirectMirror, + AutonomousClock { heartbeat_interval: Duration }, +} + +impl ShadowServiceModeV1 { + fn is_autonomous(self) -> bool { + matches!(self, Self::AutonomousClock { .. }) + } + + fn heartbeat_interval(self) -> Option { + match self { + Self::DirectMirror => None, + Self::AutonomousClock { heartbeat_interval } => Some(heartbeat_interval), + } + } + + fn carrier_sync_grace_interval(self) -> Duration { + self.heartbeat_interval() + .map(|interval| interval.saturating_mul(2)) + .unwrap_or_default() + .max(SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1) + } +} #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct ShadowLocalCarrierV1 { @@ -87,9 +129,18 @@ enum ShadowServiceMessageV1 { peer: AuthorityIndex, response: RbcDagShadowCarrierResponse, }, + CarrierSyncRequest { + peer: AuthorityIndex, + request: RbcDagShadowCarrierSyncRequest, + }, + CarrierSyncResponse { + peer: AuthorityIndex, + response: RbcDagShadowCarrierSyncResponse, + }, DirectDeliveriesChanged, TopologyChanged, RetryRecovery, + HeartbeatTick, Shutdown(oneshot::Sender>), } @@ -100,6 +151,7 @@ pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { own_authority: AuthorityIndex, committee_size: usize, input_capacity: usize, + mode: ShadowServiceModeV1, desired_topology: Arc>>, desired_direct_deliveries: Arc>>, invalidated_by_overload: Arc>>, @@ -127,6 +179,12 @@ impl StarfishRbcDagShadowServiceHandleV1 { &self, header: &RbcCanonicalHeader, ) -> Result<(), ShadowServiceErrorV1> { + if self.mode.is_autonomous() { + // The autonomous carrier clock is deliberately independent from + // direct consensus rounds. Direct headers continue through the + // authoritative path and cannot consume carrier slots. + return Ok(()); + } self.send(ShadowServiceMessageV1::LocalCarrier( ShadowLocalCarrierV1::from_direct_header(header), )) @@ -171,10 +229,45 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierResponse { peer, response }) } + pub(crate) fn carrier_sync_request( + &self, + peer: AuthorityIndex, + request: RbcDagShadowCarrierSyncRequest, + ) -> Result<(), ShadowServiceErrorV1> { + if !self.mode.is_autonomous() { + return Ok(()); + } + self.send(ShadowServiceMessageV1::CarrierSyncRequest { peer, request }) + } + + pub(crate) fn carrier_sync_response( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierSyncResponse, + ) -> Result<(), ShadowServiceErrorV1> { + if !self.mode.is_autonomous() { + return Ok(()); + } + validate_wire_size( + "carrier sync response", + response.canonical_carrier.len(), + MAX_CARRIER_CONTENT_SIZE_V1, + )?; + validate_wire_size( + "carrier sync authentication sidecar", + response.authentication_sidecar.len(), + self.max_sidecar_size, + )?; + self.send(ShadowServiceMessageV1::CarrierSyncResponse { peer, response }) + } + pub(crate) fn direct_delivered( &self, identity: ShadowDeliveryIdentityV1, ) -> Result<(), ShadowServiceErrorV1> { + if self.mode.is_autonomous() { + return Ok(()); + } if identity.author as usize >= self.committee_size { return Err(ShadowServiceErrorV1::UnknownAuthority(identity.author)); } @@ -245,9 +338,12 @@ impl ShadowServiceMessageV1 { Self::Carrier { .. } => "carrier", Self::CarrierRequest { .. } => "carrier_request", Self::CarrierResponse { .. } => "carrier_response", + Self::CarrierSyncRequest { .. } => "carrier_sync_request", + Self::CarrierSyncResponse { .. } => "carrier_sync_response", Self::DirectDeliveriesChanged => "direct_deliveries_changed", Self::TopologyChanged => "topology_changed", Self::RetryRecovery => "recovery_retry", + Self::HeartbeatTick => "heartbeat_tick", Self::Shutdown(_) => "shutdown", } } @@ -255,7 +351,16 @@ impl ShadowServiceMessageV1 { #[derive(Debug)] pub(crate) enum ShadowServiceEventV1 { - Ready, + Ready { + autonomous_clock: bool, + }, + ClockState { + open_round: RoundNumber, + phase_backlog: usize, + admitted_authors: usize, + admitted_stake: u64, + buffered_authenticated: usize, + }, ComparisonBacklog { unpaired_direct: usize, unpaired_shadow: usize, @@ -305,6 +410,7 @@ pub(crate) enum ShadowServiceErrorV1 { }, CommitteeBurstTooLarge { committee_size: usize, + required_capacity: usize, maximum_capacity: usize, }, UnknownAuthority(AuthorityIndex), @@ -312,6 +418,7 @@ pub(crate) enum ShadowServiceErrorV1 { ConflictingLocalHeader(RoundNumber), MissingRecoveredLocalHeader(RoundNumber), RecoveredLocalHeaderMismatch(RoundNumber), + AutonomousWalContainsApplicationCarrier(RoundNumber), LocalHeaderAuthority { expected: AuthorityIndex, actual: AuthorityIndex, @@ -322,6 +429,21 @@ pub(crate) enum ShadowServiceErrorV1 { peer: AuthorityIndex, reference: BlockReference, }, + InvalidHeartbeatInterval, + SyncRequestForForeignAuthor { + expected: AuthorityIndex, + actual: AuthorityIndex, + }, + UnexpectedSyncResponse { + author: AuthorityIndex, + round: RoundNumber, + }, + SyncResponseSlotMismatch { + expected_author: AuthorityIndex, + expected_round: RoundNumber, + actual_author: AuthorityIndex, + actual_round: RoundNumber, + }, } impl fmt::Display for ShadowServiceErrorV1 { @@ -352,14 +474,12 @@ impl fmt::Display for ShadowServiceErrorV1 { ), Self::CommitteeBurstTooLarge { committee_size, + required_capacity, maximum_capacity, } => write!( formatter, "Starfish-RBC-DAG shadow committee size {committee_size} needs a burst queue of \ - {}, above the memory-safe capacity limit {maximum_capacity}", - committee_size - .saturating_sub(1) - .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1), + {required_capacity}, above the memory-safe capacity limit {maximum_capacity}", ), Self::UnknownAuthority(authority) => { write!(formatter, "unknown shadow peer authority {authority}") @@ -379,6 +499,10 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "persisted shadow carrier and recovered direct header disagree at round {round}" ), + Self::AutonomousWalContainsApplicationCarrier(round) => write!( + formatter, + "autonomous carrier-clock WAL contains a non-heartbeat local carrier at round {round}" + ), Self::LocalHeaderAuthority { expected, actual } => write!( formatter, "shadow local header authority {actual} does not match local authority {expected}" @@ -393,6 +517,26 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "shadow response for {reference} came from non-holder {peer}" ), + Self::InvalidHeartbeatInterval => formatter.write_str( + "Starfish-RBC-DAG autonomous heartbeat interval must be nonzero", + ), + Self::SyncRequestForForeignAuthor { expected, actual } => write!( + formatter, + "shadow carrier sync request asked authority {expected} to serve authority {actual}" + ), + Self::UnexpectedSyncResponse { author, round } => write!( + formatter, + "unexpected shadow carrier sync response for authority {author} round {round}" + ), + Self::SyncResponseSlotMismatch { + expected_author, + expected_round, + actual_author, + actual_round, + } => write!( + formatter, + "shadow carrier sync response for authority {expected_author} round {expected_round} contained authority {actual_author} round {actual_round}" + ), } } } @@ -427,9 +571,65 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( JoinHandle<()>, ), ShadowServiceErrorV1, +> { + start_starfish_rbc_dag_shadow_service_with_mode_v1( + path, + committee, + own_authority, + context, + authorizer, + recovered_local_headers, + ShadowServiceModeV1::DirectMirror, + ) +} + +pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + heartbeat_interval: Duration, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, +> { + if heartbeat_interval.is_zero() { + return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); + } + start_starfish_rbc_dag_shadow_service_with_mode_v1( + path, + committee, + own_authority, + context, + authorizer, + Vec::new(), + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + ) +} + +fn start_starfish_rbc_dag_shadow_service_with_mode_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, + mode: ShadowServiceModeV1, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, > { let committee_size = committee.committee().len(); - let input_capacity = shadow_input_capacity(committee_size)?; + let input_capacity = shadow_input_capacity(committee_size, mode)?; let max_sidecar_size = authentication_sidecar_size(context.authentication_scheme(), committee_size); let path = path.as_ref().to_path_buf(); @@ -453,9 +653,12 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); let invalidated_by_overload = Arc::new(Mutex::new(None)); + let retry_notification_pending = Arc::new(AtomicBool::new(false)); + let heartbeat_notification_pending = Arc::new(AtomicBool::new(false)); let retry_tx = message_tx.downgrade(); + let retry_pending = Arc::clone(&retry_notification_pending); tokio::spawn(async move { - let mut interval = tokio::time::interval(SHADOW_RECOVERY_RETRY_INTERVAL_V1); + let mut interval = tokio::time::interval(SHADOW_MAINTENANCE_INTERVAL_V1); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); interval.tick().await; loop { @@ -463,16 +666,56 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( let Some(retry_tx) = retry_tx.upgrade() else { break; }; + if retry_pending.swap(true, Ordering::AcqRel) { + continue; + } match retry_tx.try_send(ShadowServiceMessageV1::RetryRecovery) { - Ok(()) | Err(TrySendError::Full(_)) => {} - Err(TrySendError::Closed(_)) => break, + Ok(()) => {} + Err(TrySendError::Full(_)) => retry_pending.store(false, Ordering::Release), + Err(TrySendError::Closed(_)) => { + retry_pending.store(false, Ordering::Release); + break; + } } } }); + if let Some(heartbeat_interval) = mode.heartbeat_interval() { + let heartbeat_tx = message_tx.downgrade(); + let heartbeat_pending = Arc::clone(&heartbeat_notification_pending); + tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Give startup/WAL replay one full interval before the first + // carrier. A missed/full notification is harmless: a later tick + // retries the still-open local slot. + interval.tick().await; + loop { + interval.tick().await; + let Some(heartbeat_tx) = heartbeat_tx.upgrade() else { + break; + }; + if heartbeat_pending.swap(true, Ordering::AcqRel) { + continue; + } + match heartbeat_tx.try_send(ShadowServiceMessageV1::HeartbeatTick) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + heartbeat_pending.store(false, Ordering::Release); + } + Err(TrySendError::Closed(_)) => { + heartbeat_pending.store(false, Ordering::Release); + break; + } + } + } + }); + } let startup_events = event_tx.clone(); let actor_desired_topology = Arc::clone(&desired_topology); let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); + let actor_retry_notification_pending = Arc::clone(&retry_notification_pending); + let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); let task = tokio::spawn(async move { let opened = tokio::task::spawn_blocking(move || { StarfishRbcDagShadowV1::open(path, committee, own_authority, context, authorizer) @@ -502,8 +745,8 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( let persisted_local = match core.local_outbound_metadata() { Ok(metadata) => metadata .into_iter() - .map(|(round, commitment, creation_time_ns)| { - (round, (commitment, creation_time_ns)) + .map(|(round, commitment, creation_time_ns, control_shape)| { + (round, (commitment, creation_time_ns, control_shape)) }) .collect::>(), Err(error) => { @@ -517,43 +760,66 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( } }; let durable_round = core.local_carrier_round(); - for (round, (commitment, creation_time_ns)) in &persisted_local { - let Some(recovered) = pending_local.get(round) else { + if mode.is_autonomous() { + if let Some((round, _)) = + persisted_local + .iter() + .find(|(_, (commitment, _, control_shape))| { + *commitment != crate::crypto::TransactionsCommitment::default() + || !*control_shape + }) + { let _ = startup_events .send(ShadowServiceEventV1::Rejected { peer: None, - error: ShadowServiceErrorV1::MissingRecoveredLocalHeader(*round) - .to_string(), + error: ShadowServiceErrorV1::AutonomousWalContainsApplicationCarrier( + *round, + ) + .to_string(), }) .await; return; - }; - if recovered.transactions_commitment != *commitment - || recovered.creation_time_ns != *creation_time_ns + } + } else { + for (round, (commitment, creation_time_ns, _)) in &persisted_local { + let Some(recovered) = pending_local.get(round) else { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::MissingRecoveredLocalHeader(*round) + .to_string(), + }) + .await; + return; + }; + if recovered.transactions_commitment != *commitment + || recovered.creation_time_ns != *creation_time_ns + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) + .to_string(), + }) + .await; + return; + } + } + if let Some(round) = pending_local + .keys() + .copied() + .find(|round| *round < durable_round && !persisted_local.contains_key(round)) { let _ = startup_events .send(ShadowServiceEventV1::Rejected { peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(round) .to_string(), }) .await; return; } } - if let Some(round) = pending_local - .keys() - .copied() - .find(|round| *round < durable_round && !persisted_local.contains_key(round)) - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(round).to_string(), - }) - .await; - return; - } let reported_shadow_deliveries = match core.delivered_identities() { Ok(identities) => identities.into_iter().collect::>(), Err(error) => { @@ -573,8 +839,10 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( .collect(); let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); pending_local.retain(|round, _| *round >= core.local_carrier_round()); + let sync_round = core.local_carrier_round(); let state = ShadowServiceStateV1 { core, + mode, own_authority, committee_size, events: event_tx, @@ -586,6 +854,14 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( pending_local, pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), + sync_last_attempt: BTreeMap::new(), + sync_last_served: BTreeMap::new(), + sync_round, + sync_round_opened_at: Instant::now(), + sync_catch_up: false, + sync_used_in_open_round: false, + retry_notification_pending: actor_retry_notification_pending, + heartbeat_notification_pending: actor_heartbeat_notification_pending, direct_deliveries: BTreeSet::new(), reported_shadow_deliveries, recovered_shadow_deliveries, @@ -615,6 +891,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( own_authority, committee_size, input_capacity, + mode, desired_topology, desired_direct_deliveries, invalidated_by_overload, @@ -690,6 +967,7 @@ impl ShadowComparisonBacklogV1 { struct ShadowServiceStateV1 { core: StarfishRbcDagShadowV1, + mode: ShadowServiceModeV1, own_authority: AuthorityIndex, committee_size: usize, events: mpsc::Sender, @@ -701,6 +979,14 @@ struct ShadowServiceStateV1 { pending_local: BTreeMap, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, + sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, + sync_last_served: BTreeMap, + sync_round: RoundNumber, + sync_round_opened_at: Instant, + sync_catch_up: bool, + sync_used_in_open_round: bool, + retry_notification_pending: Arc, + heartbeat_notification_pending: Arc, direct_deliveries: BTreeSet, reported_shadow_deliveries: BTreeSet, recovered_shadow_deliveries: BTreeSet, @@ -724,6 +1010,9 @@ impl ShadowServiceStateV1 { } fn emit_comparison_backlog(&self) { + if self.mode.is_autonomous() { + return; + } let (unpaired_direct, unpaired_shadow, max_round_lag) = self.comparison_backlog.counts(); self.emit(ShadowServiceEventV1::ComparisonBacklog { unpaired_direct, @@ -732,6 +1021,19 @@ impl ShadowServiceStateV1 { }); } + fn emit_clock_state(&self) { + if !self.mode.is_autonomous() { + return; + } + self.emit(ShadowServiceEventV1::ClockState { + open_round: self.core.local_carrier_round(), + phase_backlog: self.core.pending_phase_backlog_len(), + admitted_authors: self.core.current_round_admitted_author_count(), + admitted_stake: self.core.current_round_admitted_stake(), + buffered_authenticated: self.core.buffered_authenticated_carrier_count(), + }); + } + fn validate_peer(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { if peer as usize >= self.committee_size { return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); @@ -751,6 +1053,10 @@ impl ShadowServiceStateV1 { } self.recovery_last_attempt .retain(|(_, holder), _| holder != peer); + self.sync_last_attempt + .retain(|(author, _), _| author != peer); + self.sync_last_served + .retain(|requester, _| requester != peer); if state.0 { self.connected.insert(*peer); newly_connected.push(*peer); @@ -760,10 +1066,17 @@ impl ShadowServiceStateV1 { } self.observed_topology = desired; if !newly_connected.is_empty() { - let retransmissions = self.core.retransmissions(); - for peer in newly_connected { - for envelope in &retransmissions { - self.send_envelope(peer, envelope); + if self.mode.is_autonomous() { + // Autonomous history is synchronized one exact slot at a + // time. Replaying the entire retained run on every reconnect + // would create an unbounded burst as heartbeats accumulate. + self.flush_carrier_sync_requests(self.core.local_carrier_round() > 1); + } else { + let retransmissions = self.core.retransmissions(); + for peer in newly_connected { + for envelope in &retransmissions { + self.send_envelope(peer, envelope); + } } } self.flush_recovery_requests(); @@ -771,6 +1084,9 @@ impl ShadowServiceStateV1 { } fn reconcile_direct_deliveries(&mut self) { + if self.mode.is_autonomous() { + return; + } let desired = self.desired_direct_deliveries.lock().clone(); let newly_observed = desired .difference(&self.direct_deliveries) @@ -845,6 +1161,12 @@ impl ShadowServiceStateV1 { } fn process_effects(&mut self, effects: Vec) { + let carrier_round_advanced = effects + .iter() + .any(|effect| matches!(effect, ModelEffect::CarrierRoundAdvanced(_))); + if carrier_round_advanced { + self.sync_catch_up = std::mem::take(&mut self.sync_used_in_open_round); + } for effect in effects { match effect { ModelEffect::NeedCarrier { target, holders } => { @@ -858,7 +1180,8 @@ impl ShadowServiceStateV1 { ModelEffect::Delivered(reference) => { self.pending_recovery.remove(&reference); } - ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} + ModelEffect::PrefixAdvanced { .. } => {} + ModelEffect::CarrierRoundAdvanced(_) => {} } } self.reconcile_pending_recovery(); @@ -867,6 +1190,59 @@ impl ShadowServiceStateV1 { self.pending_recovery.len(), )); self.report_new_shadow_deliveries(); + self.flush_carrier_sync_requests(false); + self.emit_clock_state(); + } + + fn try_create_autonomous_heartbeat(&mut self) { + if !self.mode.is_autonomous() || !self.core.can_create_carrier() { + self.emit_clock_state(); + return; + } + let creation_time_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .try_into() + .unwrap_or(TimestampNs::MAX); + let before = self.core.wal_counts(); + match self.core.create_local_control_heartbeat(creation_time_ns) { + Ok((envelope, effects)) => { + self.emit(ShadowServiceEventV1::Input { + kind: "heartbeat", + outcome: "accepted", + }); + self.report_wal_delta(before); + self.broadcast(&envelope); + self.process_effects(effects); + } + Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { + // The local slot is open syntactically but cannot yet name a + // quorum of exact previous-round admitted parents. A later + // authenticated ingress or timer tick retries it. + self.emit(ShadowServiceEventV1::Input { + kind: "heartbeat", + outcome: "waiting_for_quorum", + }); + self.emit_clock_state(); + } + Err(error) => self.mark_fatal(error), + } + } + + /// While repairing a lagging clock, fix our next control carrier as soon + /// as its exact previous-round quorum is available. Waiting for the + /// normal heartbeat interval would cap catch-up at the production rate, + /// so a node behind a continuously advancing committee could never close + /// the gap. Healthy rounds still remain paced exclusively by the timer. + fn drive_autonomous_catch_up(&mut self) { + while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { + let round_before = self.core.local_carrier_round(); + self.try_create_autonomous_heartbeat(); + if self.core.local_carrier_round() == round_before { + break; + } + } } fn reconcile_pending_recovery(&mut self) { @@ -1000,6 +1376,231 @@ impl ShadowServiceStateV1 { } } + fn flush_carrier_sync_requests(&mut self, force: bool) { + if !self.mode.is_autonomous() { + return; + } + let round = self.core.local_carrier_round(); + let now = Instant::now(); + if round != self.sync_round { + self.sync_round = round; + self.sync_round_opened_at = now; + self.sync_last_attempt.clear(); + } + self.sync_last_attempt.retain(|(author, attempt_round), _| { + *attempt_round == round + && self.connected.contains(author) + && self.core.admitted_reference(*author, round).is_none() + }); + if !force + && !self.sync_catch_up + && now.saturating_duration_since(self.sync_round_opened_at) + < self.mode.carrier_sync_grace_interval() + { + return; + } + let requests = self + .connected + .iter() + .copied() + .filter(|author| self.core.admitted_reference(*author, round).is_none()) + .filter(|author| { + self.sync_last_attempt + .get(&(*author, round)) + .is_none_or(|last| { + now.saturating_duration_since(*last) + >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 + }) + }) + .collect::>(); + for author in requests { + self.sync_last_attempt.insert((author, round), now); + self.emit(ShadowServiceEventV1::Network { + recipient: author, + message: NetworkMessage::RbcDagShadowCarrierSyncRequest( + RbcDagShadowCarrierSyncRequest { author, round }, + ), + }); + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "sent", + }); + } + } + + fn handle_carrier_sync_request( + &mut self, + peer: AuthorityIndex, + request: RbcDagShadowCarrierSyncRequest, + ) { + if request.author != self.own_authority { + self.reject( + Some(peer), + ShadowServiceErrorV1::SyncRequestForForeignAuthor { + expected: self.own_authority, + actual: request.author, + }, + ); + return; + } + if request.round > self.core.local_carrier_round() { + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "future_not_found", + }); + return; + } + let now = Instant::now(); + if self.sync_last_served.get(&peer).is_some_and(|(_, last)| { + now.saturating_duration_since(*last) < SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 + }) { + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "rate_limited", + }); + return; + } + self.sync_last_served.insert(peer, (request.round, now)); + let Some(envelope) = self.core.local_outbound_envelope(request.round) else { + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "not_found", + }); + return; + }; + self.emit(ShadowServiceEventV1::Network { + recipient: peer, + message: NetworkMessage::RbcDagShadowCarrierSyncResponse( + RbcDagShadowCarrierSyncResponse { + author: request.author, + round: request.round, + canonical_carrier: envelope.canonical_carrier_wire().to_vec(), + authentication_sidecar: envelope.authentication_sidecar().to_vec(), + }, + ), + }); + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "served", + }); + } + + fn handle_carrier_sync_response( + &mut self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierSyncResponse, + ) { + let expected = (response.author, response.round); + if peer != response.author { + self.reject( + Some(peer), + ShadowServiceErrorV1::UnexpectedSyncResponse { + author: response.author, + round: response.round, + }, + ); + return; + } + let (actual_author, actual_round, actual_reference) = + match self.core.candidate_slot(&response.canonical_carrier) { + Ok(slot) => slot, + Err(error) => { + self.reject(Some(peer), error); + return; + } + }; + if actual_author != response.author || actual_round != response.round { + self.reject( + Some(peer), + ShadowServiceErrorV1::SyncResponseSlotMismatch { + expected_author: response.author, + expected_round: response.round, + actual_author, + actual_round, + }, + ); + return; + } + if response.round < self.core.local_carrier_round() + || self + .core + .admitted_reference(response.author, response.round) + .is_some() + { + self.sync_last_attempt.remove(&expected); + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_response", + outcome: "ignored_already_admitted_or_stale", + }); + return; + } + if !self.sync_last_attempt.contains_key(&expected) { + self.reject( + Some(peer), + ShadowServiceErrorV1::UnexpectedSyncResponse { + author: response.author, + round: response.round, + }, + ); + return; + } + let before = self.core.wal_counts(); + match self.core.receive_or_retain_from_peer( + &response.canonical_carrier, + &response.authentication_sidecar, + peer, + ) { + Ok(outcome) => { + let outcome_label = match outcome.disposition() { + ShadowIngressDispositionV1::Authenticated => "authenticated", + ShadowIngressDispositionV1::CandidateRetained => "retained_unauthenticated", + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => "ignored", + }; + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_response", + outcome: outcome_label, + }); + if outcome.disposition() == ShadowIngressDispositionV1::Authenticated + && self + .core + .admitted_reference(response.author, response.round) + == Some(actual_reference) + { + self.sync_used_in_open_round = true; + } + self.report_wal_delta(before); + self.process_effects(outcome.effects().to_vec()); + if self + .core + .admitted_reference(response.author, response.round) + .is_some() + { + self.sync_last_attempt.remove(&expected); + } + if self.sync_catch_up { + self.flush_carrier_sync_requests(true); + } + if outcome.disposition() == ShadowIngressDispositionV1::CandidateRetained { + self.reject( + Some(peer), + ShadowServiceErrorV1::UnauthenticatedCarrierRetained, + ); + } + } + Err(error) => { + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_response", + outcome: "rejected", + }); + if is_fatal_core_error(&error) { + self.mark_fatal(error); + } else { + self.reject(Some(peer), error); + } + } + } + } + fn report_new_shadow_deliveries(&mut self) { let identities: BTreeSet<_> = match self.core.delivered_identities() { Ok(identities) => identities.into_iter().collect(), @@ -1015,7 +1616,9 @@ impl ShadowServiceStateV1 { self.reported_shadow_deliveries = identities; for identity in &new_identities { let slot = delivery_slot(identity); - self.comparison_backlog.observe_epoch_shadow(slot); + if !self.mode.is_autonomous() { + self.comparison_backlog.observe_epoch_shadow(slot); + } self.emit(ShadowServiceEventV1::Delivered(*identity)); self.emit_slot_comparison(slot); self.emit_comparison_backlog(); @@ -1023,6 +1626,9 @@ impl ShadowServiceStateV1 { } fn emit_slot_comparison(&mut self, slot: ShadowDeliverySlotV1) { + if self.mode.is_autonomous() { + return; + } let direct = self .direct_deliveries .iter() @@ -1085,10 +1691,16 @@ fn run_shadow_service( state.reconcile_topology(); state.reconcile_direct_deliveries(); if !state.observe_external_invalidation() { - state.emit(ShadowServiceEventV1::Ready); + state.emit(ShadowServiceEventV1::Ready { + autonomous_clock: state.mode.is_autonomous(), + }); state.emit_comparison_backlog(); state.process_effects(open_report.recovery_effects().to_vec()); - state.retry_pending_local(); + if state.mode.is_autonomous() { + state.emit_clock_state(); + } else { + state.retry_pending_local(); + } } while !state.fatal { @@ -1116,6 +1728,15 @@ fn run_shadow_service( if state.observe_external_invalidation() { break; } + match &message { + ShadowServiceMessageV1::RetryRecovery => state + .retry_notification_pending + .store(false, Ordering::Release), + ShadowServiceMessageV1::HeartbeatTick => state + .heartbeat_notification_pending + .store(false, Ordering::Release), + _ => {} + } match message { ShadowServiceMessageV1::LocalCarrier(local) => { state.enqueue_local(local); @@ -1190,6 +1811,17 @@ fn run_shadow_service( state.reject(Some(peer), error); continue; } + if state + .core + .retained_candidate_wire(response.reference) + .is_some() + { + state.emit(ShadowServiceEventV1::Input { + kind: "recovery", + outcome: "ignored_already_retained", + }); + continue; + } let Some(holders) = state.pending_recovery.get(&response.reference) else { state.reject( Some(peer), @@ -1238,6 +1870,20 @@ fn run_shadow_service( } } } + ShadowServiceMessageV1::CarrierSyncRequest { peer, request } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + state.handle_carrier_sync_request(peer, request); + } + ShadowServiceMessageV1::CarrierSyncResponse { peer, response } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + state.handle_carrier_sync_response(peer, response); + } ShadowServiceMessageV1::DirectDeliveriesChanged => { state.reconcile_direct_deliveries(); } @@ -1246,11 +1892,15 @@ fn run_shadow_service( state.reconcile_topology(); state.reconcile_pending_recovery(); state.flush_recovery_requests(); + state.flush_carrier_sync_requests(false); } + ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_heartbeat(), ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); state.reconcile_direct_deliveries(); + state.drive_autonomous_catch_up(); + state.flush_carrier_sync_requests(false); } let events = state.events.clone(); if let Err(error) = state.core.shutdown() { @@ -1279,13 +1929,19 @@ fn authentication_sidecar_size(scheme: BlockAuthenticationScheme, committee_size } } -fn shadow_input_capacity(committee_size: usize) -> Result { - let committee_burst = committee_size - .saturating_sub(1) +fn shadow_input_capacity( + committee_size: usize, + mode: ShadowServiceModeV1, +) -> Result { + let peer_count = committee_size.saturating_sub(1); + let peer_burst_factor = if mode.is_autonomous() { 3 } else { 1 }; + let committee_burst = peer_count + .saturating_mul(peer_burst_factor) .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1); if committee_burst > SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1 { return Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { committee_size, + required_capacity: committee_burst, maximum_capacity: SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1, }); } @@ -1349,7 +2005,11 @@ mod tests { impl Harness { fn new() -> Self { - let committee = Committee::new_test(vec![1; N]); + Self::new_with_n(N) + } + + fn new_with_n(n: usize) -> Self { + let committee = Committee::new_test(vec![1; n]); let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); let context = RbcDagContextV1::new_with_committee( RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), @@ -1357,14 +2017,14 @@ mod tests { BlockAuthenticationScheme::MacVector, ); let directory = tempfile::tempdir().unwrap(); - let paths = (0..N) + let paths = (0..n) .map(|authority| directory.path().join(format!("shadow-{authority}.wal"))) .collect(); Self { _directory: directory, committee, context, - keyrings: mac_keyrings_for_test(N), + keyrings: mac_keyrings_for_test(n), paths, } } @@ -1389,6 +2049,37 @@ mod tests { .unwrap() } + fn start_autonomous( + &self, + authority: AuthorityIndex, + ) -> ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ) { + self.start_autonomous_with_interval(authority, Duration::from_secs(60 * 60)) + } + + fn start_autonomous_with_interval( + &self, + authority: AuthorityIndex, + heartbeat_interval: Duration, + ) -> ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ) { + start_starfish_rbc_dag_autonomous_clock_service_v1( + &self.paths[authority as usize], + self.committee.clone(), + authority, + self.context, + ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), + heartbeat_interval, + ) + .unwrap() + } + fn envelope( &self, candidate: &CandidateCarrierV1, @@ -1422,7 +2113,7 @@ mod tests { async fn wait_ready(events: &mut mpsc::Receiver) { loop { match next_event(events).await { - ShadowServiceEventV1::Ready => return, + ShadowServiceEventV1::Ready { .. } => return, ShadowServiceEventV1::Rejected { error, .. } => { panic!("shadow startup failed: {error}") } @@ -1467,6 +2158,175 @@ mod tests { } } + async fn pump_autonomous_until_round( + handles: &[StarfishRbcDagShadowServiceHandleV1], + events: &mut [mpsc::Receiver], + open_rounds: &mut [RoundNumber], + deliveries: &mut [usize], + sync_requests: &mut usize, + target_open_round: RoundNumber, + ) { + timeout(EVENT_TIMEOUT, async { + loop { + let mut progressed = false; + for sender in 0..events.len() { + while let Ok(event) = events[sender].try_recv() { + progressed = true; + match event { + ShadowServiceEventV1::Network { recipient, message } => { + let recipient = recipient as usize; + match message { + NetworkMessage::RbcDagShadowCarrier(envelope) => handles + [recipient] + .carrier(sender as AuthorityIndex, envelope) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierRequest(reference) => handles + [recipient] + .carrier_request(sender as AuthorityIndex, reference) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierResponse(response) => handles + [recipient] + .carrier_response(sender as AuthorityIndex, response) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { + *sync_requests = sync_requests.saturating_add(1); + handles[recipient] + .carrier_sync_request( + sender as AuthorityIndex, + request, + ) + .unwrap(); + } + NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { + handles[recipient] + .carrier_sync_response( + sender as AuthorityIndex, + response, + ) + .unwrap(); + } + unexpected => panic!( + "autonomous shadow emitted unexpected network message: {unexpected:?}" + ), + } + } + ShadowServiceEventV1::ClockState { open_round, .. } => { + open_rounds[sender] = open_rounds[sender].max(open_round); + } + ShadowServiceEventV1::Delivered(_) => { + deliveries[sender] = deliveries[sender].saturating_add(1); + } + ShadowServiceEventV1::Rejected { error, .. } + if error.contains("FutureCarrierOutsideBuffer") + || error.contains("unexpected shadow response") => {} + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("autonomous shadow rejected valid test traffic: {error}") + } + _ => {} + } + } + } + if open_rounds + .iter() + .all(|round| *round >= target_open_round) + { + return; + } + if !progressed { + tokio::time::sleep(Duration::from_millis(1)).await; + } else { + tokio::task::yield_now().await; + } + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "autonomous clock did not open round {target_open_round}; open={open_rounds:?}, sync_requests={sync_requests}" + ) + }); + } + + async fn pump_online_prefix_until_round( + handles: &[StarfishRbcDagShadowServiceHandleV1], + events: &mut [mpsc::Receiver], + open_rounds: &mut [RoundNumber], + online: usize, + target_open_round: RoundNumber, + ) { + timeout(EVENT_TIMEOUT, async { + loop { + let mut progressed = false; + for sender in 0..events.len() { + while let Ok(event) = events[sender].try_recv() { + progressed = true; + match event { + ShadowServiceEventV1::Network { recipient, message } + if sender < online && (recipient as usize) < online => + { + let recipient = recipient as usize; + match message { + NetworkMessage::RbcDagShadowCarrier(envelope) => handles + [recipient] + .carrier(sender as AuthorityIndex, envelope) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierRequest(reference) => handles + [recipient] + .carrier_request(sender as AuthorityIndex, reference) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierResponse(response) => handles + [recipient] + .carrier_response(sender as AuthorityIndex, response) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { + handles[recipient] + .carrier_sync_request( + sender as AuthorityIndex, + request, + ) + .unwrap(); + } + NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { + handles[recipient] + .carrier_sync_response( + sender as AuthorityIndex, + response, + ) + .unwrap(); + } + unexpected => panic!( + "autonomous shadow emitted unexpected network message: {unexpected:?}" + ), + } + } + ShadowServiceEventV1::Network { .. } => {} + ShadowServiceEventV1::ClockState { open_round, .. } => { + open_rounds[sender] = open_rounds[sender].max(open_round); + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("autonomous shadow rejected valid test traffic: {error}") + } + _ => {} + } + } + } + if open_rounds[..online] + .iter() + .all(|round| *round >= target_open_round) + { + return; + } + if !progressed { + tokio::time::sleep(Duration::from_millis(1)).await; + } else { + tokio::task::yield_now().await; + } + } + }) + .await + .unwrap_or_else(|_| panic!("online prefix did not open round {target_open_round}")); + } + async fn stop( handle: StarfishRbcDagShadowServiceHandleV1, events: mpsc::Receiver, @@ -1481,7 +2341,9 @@ mod tests { loop { match next_event(&mut events).await { ShadowServiceEventV1::Rejected { peer: None, error } => return error, - ShadowServiceEventV1::Ready => panic!("invalid shadow startup became ready"), + ShadowServiceEventV1::Ready { .. } => { + panic!("invalid shadow startup became ready") + } _ => {} } } @@ -1584,6 +2446,350 @@ mod tests { stop(handle, events, task).await; } + async fn assert_autonomous_zero_load_progress(n: usize) { + let harness = Harness::new_with_n(n); + let mut handles = Vec::new(); + let mut events = Vec::new(); + let mut tasks = Vec::new(); + for authority in 0..n as AuthorityIndex { + let (handle, mut node_events, task) = harness.start_autonomous(authority); + loop { + match next_event(&mut node_events).await { + ShadowServiceEventV1::Ready { autonomous_clock } => { + assert!(autonomous_clock); + break; + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("autonomous shadow startup failed: {error}") + } + _ => {} + } + } + handles.push(handle); + events.push(node_events); + tasks.push(task); + } + + let mut open_rounds = vec![1; n]; + let mut deliveries = vec![0; n]; + let mut sync_requests = 0; + for (authority, handle) in handles.iter().enumerate() { + for peer in 0..n { + if peer != authority { + handle.peer_connected(peer as AuthorityIndex).unwrap(); + } + } + } + for fixed_round in 1..=6 { + for handle in &handles { + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + } + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut sync_requests, + fixed_round + 1, + ) + .await; + } + + assert!( + deliveries.iter().all(|count| *count > 0), + "every node must RBC-deliver mature heartbeat carriers: {deliveries:?}" + ); + assert!(open_rounds.iter().all(|round| *round >= 7)); + assert_eq!( + sync_requests, 0, + "healthy proactive rounds must not trigger repair polling" + ); + + drop(events); + for handle in &handles { + handle.shutdown().await.unwrap(); + } + for task in tasks { + task.await.unwrap(); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn four_node_autonomous_zero_load_clock_delivers_mature_heartbeats() { + assert_autonomous_zero_load_progress(4).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn seven_node_autonomous_zero_load_clock_delivers_mature_heartbeats() { + assert_autonomous_zero_load_progress(7).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn autonomous_exact_slot_sync_is_bounded_and_late_response_is_idempotent() { + let harness = Harness::new(); + let heartbeat_interval = Duration::from_millis(250); + let (author, mut author_events, author_task) = + harness.start_autonomous_with_interval(0, heartbeat_interval); + let (receiver, mut receiver_events, receiver_task) = + harness.start_autonomous_with_interval(1, heartbeat_interval); + loop { + if let ShadowServiceEventV1::Ready { autonomous_clock } = + next_event(&mut author_events).await + { + assert!(autonomous_clock); + break; + } + } + loop { + if let ShadowServiceEventV1::Ready { autonomous_clock } = + next_event(&mut receiver_events).await + { + assert!(autonomous_clock); + break; + } + } + + author.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + let proactive = next_carrier(&mut author_events, 1).await; + receiver.peer_connected(0).unwrap(); + tokio::time::sleep(SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1).await; + + let request = loop { + if let ShadowServiceEventV1::Network { + recipient: 0, + message: NetworkMessage::RbcDagShadowCarrierSyncRequest(request), + } = next_event(&mut receiver_events).await + { + break request; + } + }; + assert_eq!((request.author, request.round), (0, 1)); + author.carrier_sync_request(1, request).unwrap(); + let response = loop { + if let ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrierSyncResponse(response), + } = next_event(&mut author_events).await + { + break response; + } + }; + assert_eq!(response.canonical_carrier, proactive.canonical_carrier); + assert_eq!( + response.authentication_sidecar, + proactive.authentication_sidecar + ); + receiver.carrier_sync_response(0, response.clone()).unwrap(); + loop { + match next_event(&mut receiver_events).await { + ShadowServiceEventV1::Input { + kind: "carrier_sync_response", + outcome: "authenticated", + } => break, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("valid exact-slot response was rejected: {error}") + } + _ => {} + } + } + + // A proactive/response race can leave an authenticated response in + // flight after the slot was admitted. It is an idempotent replay, not + // peer misbehavior and not a benchmark-invalidating error. + receiver.carrier_sync_response(0, response).unwrap(); + loop { + match next_event(&mut receiver_events).await { + ShadowServiceEventV1::Input { + kind: "carrier_sync_response", + outcome: "ignored_already_admitted_or_stale", + } => break, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("late exact-slot response was not idempotent: {error}") + } + _ => {} + } + } + + // One requester cannot amplify repeated reads of a retained large + // carrier faster than the bounded synchronization interval. + author.carrier_sync_request(1, request).unwrap(); + loop { + if let ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "rate_limited", + } = next_event(&mut author_events).await + { + break; + } + } + + stop(author, author_events, author_task).await; + stop(receiver, receiver_events, receiver_task).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn autonomous_exact_sync_closes_a_multi_round_gap() { + let harness = Harness::new(); + let mut handles = Vec::new(); + let mut events = Vec::new(); + let mut tasks = Vec::new(); + for authority in 0..N as AuthorityIndex { + let (handle, mut node_events, task) = harness.start_autonomous(authority); + wait_ready(&mut node_events).await; + handles.push(handle); + events.push(node_events); + tasks.push(task); + } + + // Establish round one for all validators, then let a quorum advance + // while authority 3 is offline and receives none of the proactive + // carriers. Starting the gap at round two makes reconnect request + // exact repair immediately; the one-hour normal heartbeat still + // cannot help with the later repaired rounds. + for (authority, handle) in handles.iter().enumerate() { + for peer in 0..N { + if peer != authority { + handle.peer_connected(peer as AuthorityIndex).unwrap(); + } + } + } + let mut open_rounds = vec![1; N]; + let mut deliveries = vec![0; N]; + let mut sync_requests = 0; + for handle in &handles { + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + } + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut sync_requests, + 2, + ) + .await; + for authority in 0..3 { + handles[authority].peer_disconnected(3).unwrap(); + handles[3] + .peer_disconnected(authority as AuthorityIndex) + .unwrap(); + } + for fixed_round in 2..=8 { + for handle in &handles[..3] { + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + } + pump_online_prefix_until_round( + &handles, + &mut events, + &mut open_rounds, + 3, + fixed_round + 1, + ) + .await; + } + assert_eq!(open_rounds[3], 2); + assert!(open_rounds[..3].iter().all(|round| *round >= 9)); + + // Reconnect, fix the lagging node's first local slot, and let the + // healthy quorum open one more round. Exact responses then drive an + // immediate local heartbeat per repaired round; the one-hour normal + // timer cannot be responsible for convergence. + for (authority, handle) in handles.iter().enumerate() { + for peer in 0..N { + if peer != authority { + handle.peer_connected(peer as AuthorityIndex).unwrap(); + } + } + } + handles[3] + .send(ShadowServiceMessageV1::HeartbeatTick) + .unwrap(); + for handle in &handles[..3] { + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + } + + let sync_requests_before_catch_up = sync_requests; + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut sync_requests, + 10, + ) + .await; + assert!( + sync_requests > sync_requests_before_catch_up, + "catch-up must use exact-slot repair" + ); + assert!(open_rounds.iter().all(|round| *round >= 10)); + + drop(events); + for handle in &handles { + handle.shutdown().await.unwrap(); + } + for task in tasks { + task.await.unwrap(); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn autonomous_wal_restart_serves_the_exact_persisted_heartbeat() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start_autonomous(0); + loop { + if let ShadowServiceEventV1::Ready { autonomous_clock } = next_event(&mut events).await + { + assert!(autonomous_clock); + break; + } + } + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + let original = next_carrier(&mut events, 1).await; + stop(handle, events, task).await; + + let (restarted, mut restarted_events, restarted_task) = harness.start_autonomous(0); + let mut replayed = false; + loop { + match next_event(&mut restarted_events).await { + ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches > 0, + ShadowServiceEventV1::Ready { autonomous_clock } => { + assert!(autonomous_clock); + break; + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("autonomous WAL restart failed: {error}") + } + _ => {} + } + } + assert!(replayed); + restarted + .carrier_sync_request( + 1, + RbcDagShadowCarrierSyncRequest { + author: 0, + round: 1, + }, + ) + .unwrap(); + loop { + if let ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrierSyncResponse(response), + } = next_event(&mut restarted_events).await + { + assert_eq!(response.canonical_carrier, original.canonical_carrier); + assert_eq!( + response.authentication_sidecar, + original.authentication_sidecar + ); + break; + } + } + stop(restarted, restarted_events, restarted_task).await; + } + fn phase_carrier( author: AuthorityIndex, statement: RbcPhaseStatementV1, @@ -1634,7 +2840,7 @@ mod tests { loop { match next_event(&mut restarted_events).await { ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches > 0, - ShadowServiceEventV1::Ready => break, + ShadowServiceEventV1::Ready { .. } => break, ShadowServiceEventV1::Rejected { error, .. } => { panic!("valid shadow restart failed: {error}") } @@ -1911,6 +3117,27 @@ mod tests { }; assert_eq!(identity.author, 2); assert_eq!(identity.round, 1); + handle + .carrier_response( + 1, + RbcDagShadowCarrierResponse { + reference: target.reference(), + canonical_carrier: target.canonical_wire_bytes().unwrap(), + }, + ) + .unwrap(); + loop { + match next_event(&mut events).await { + ShadowServiceEventV1::Input { + kind: "recovery", + outcome: "ignored_already_retained", + } => break, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("late exact recovery response was not idempotent: {error}") + } + _ => {} + } + } handle.direct_delivered(identity).unwrap(); loop { if let ShadowServiceEventV1::Comparison(comparison) = next_event(&mut events).await { @@ -1943,10 +3170,11 @@ mod tests { #[tokio::test] async fn bounded_input_reports_overload_but_shutdown_waits_for_capacity() { - let input_capacity = shadow_input_capacity(N).unwrap(); + let input_capacity = shadow_input_capacity(N, ShadowServiceModeV1::DirectMirror).unwrap(); let (sender, mut receiver) = mpsc::channel(input_capacity); let handle = StarfishRbcDagShadowServiceHandleV1 { sender, + mode: ShadowServiceModeV1::DirectMirror, max_sidecar_size: 3 + N * MAC_TAG_SIZE, own_authority: 0, committee_size: N, @@ -1996,6 +3224,7 @@ mod tests { let (sender, _receiver) = mpsc::channel(1); let oversized = StarfishRbcDagShadowServiceHandleV1 { sender, + mode: ShadowServiceModeV1::DirectMirror, max_sidecar_size: 3 + N * MAC_TAG_SIZE, own_authority: 0, committee_size: N, @@ -2022,12 +3251,14 @@ mod tests { #[tokio::test] async fn sixty_validator_burst_fits_before_the_actor_drains() { const LARGE_N: usize = 60; - let input_capacity = shadow_input_capacity(LARGE_N).unwrap(); + let input_capacity = + shadow_input_capacity(LARGE_N, ShadowServiceModeV1::DirectMirror).unwrap(); assert_eq!(input_capacity, SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1); let (sender, _receiver) = mpsc::channel(input_capacity); let invalidated = Arc::new(Mutex::new(None)); let handle = StarfishRbcDagShadowServiceHandleV1 { sender, + mode: ShadowServiceModeV1::DirectMirror, max_sidecar_size: 3 + LARGE_N * MAC_TAG_SIZE, own_authority: 0, committee_size: LARGE_N, @@ -2052,11 +3283,27 @@ mod tests { } assert_eq!(*invalidated.lock(), None); assert!(matches!( - shadow_input_capacity(LARGE_N + 1), + shadow_input_capacity(LARGE_N + 1, ShadowServiceModeV1::DirectMirror), Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { .. }) )); } + #[test] + fn autonomous_burst_budget_accepts_twenty_and_rejects_twenty_one() { + let mode = ShadowServiceModeV1::AutonomousClock { + heartbeat_interval: Duration::from_millis(250), + }; + assert_eq!(shadow_input_capacity(20, mode).unwrap(), 64); + assert!(matches!( + shadow_input_capacity(21, mode), + Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { + committee_size: 21, + required_capacity: 65, + maximum_capacity: 64, + }) + )); + } + #[tokio::test] async fn dropping_all_handles_stops_actor_despite_retry_timer() { let harness = Harness::new(); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index c5740afb..38744dd2 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -52,6 +52,28 @@ impl Validator { ) .map_err(|error| eyre!(error))?; let is_starfish_rbc = protocol_config.consensus_protocol.is_starfish_rbc(); + if public_config.parameters.starfish_rbc_dag_autonomous_clock + && !public_config.parameters.starfish_rbc_dag_shadow + { + return Err(eyre!( + "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" + )); + } + if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { + return Err(eyre!( + "Starfish-RBC-DAG autonomous clock requires consensus 'starfish-rbc'" + )); + } + if public_config.parameters.starfish_rbc_dag_autonomous_clock + && public_config + .parameters + .starfish_rbc_dag_heartbeat_interval_ms + == 0 + { + return Err(eyre!( + "Starfish-RBC-DAG autonomous heartbeat interval must be greater than zero" + )); + } if public_config.parameters.starfish_rbc_dag_shadow && !is_starfish_rbc { return Err(eyre!( "Starfish-RBC-DAG shadow mode requires consensus 'starfish-rbc'" @@ -200,7 +222,12 @@ impl Validator { } else { None }; - let starfish_rbc_dag_shadow_wal = private_config.starfish_rbc_dag_shadow_wal(); + let starfish_rbc_dag_shadow_wal = + if public_config.parameters.starfish_rbc_dag_autonomous_clock { + private_config.starfish_rbc_dag_autonomous_clock_wal() + } else { + private_config.starfish_rbc_dag_shadow_wal() + }; let (core, bls_cert_aggregator) = Core::open( block_handler, @@ -354,6 +381,101 @@ mod smoke_tests { })); } + #[tokio::test] + async fn autonomous_clock_requires_shadow_mode() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_autonomous_clock = true; + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("autonomous clock requires the RBC-DAG shadow") + })); + } + + #[tokio::test] + async fn autonomous_clock_rejects_non_rbc_protocol() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_shadow = true; + public_config.parameters.starfish_rbc_dag_autonomous_clock = true; + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("autonomous clock requires consensus 'starfish-rbc'") + })); + } + + #[tokio::test] + async fn autonomous_clock_rejects_zero_heartbeat_interval() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_shadow = true; + public_config.parameters.starfish_rbc_dag_autonomous_clock = true; + public_config + .parameters + .starfish_rbc_dag_heartbeat_interval_ms = 0; + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("autonomous heartbeat interval must be greater than zero") + })); + } + async fn run_commit_test( consensus: &str, block_authentication: Option<&str>, @@ -367,6 +489,23 @@ mod smoke_tests { block_authentication: Option<&str>, port_offset: u16, starfish_rbc_dag_shadow: bool, + ) { + run_commit_test_with_shadow_mode( + consensus, + block_authentication, + port_offset, + starfish_rbc_dag_shadow, + false, + ) + .await; + } + + async fn run_commit_test_with_shadow_mode( + consensus: &str, + block_authentication: Option<&str>, + port_offset: u16, + starfish_rbc_dag_shadow: bool, + autonomous_clock: bool, ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); @@ -374,6 +513,12 @@ mod smoke_tests { NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); public_config.parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; + public_config.parameters.starfish_rbc_dag_autonomous_clock = autonomous_clock; + if autonomous_clock { + public_config + .parameters + .starfish_rbc_dag_heartbeat_interval_ms = 50; + } if consensus == "starfish-rbc" { public_config .parameters @@ -421,7 +566,48 @@ mod smoke_tests { ), } - if starfish_rbc_dag_shadow { + if autonomous_clock { + tokio::time::timeout(timeout, async { + loop { + if validators.iter().all(|validator| { + let metrics = validator.metrics(); + metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1 + && metrics.starfish_rbc_dag_shadow_carrier_round.get() > 3 + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .get() + > 0 + && metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .get() + > 0 + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get() + > 0 + && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 + }) { + break; + } + time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("autonomous carrier clock did not advance while direct RBC committed"); + + for validator in &validators { + let metrics = validator.metrics(); + assert_eq!(metrics.starfish_rbc_dag_shadow_clock_valid.get(), 1); + assert!(metrics.starfish_rbc_dag_shadow_carrier_round.get() > 3); + assert_eq!( + metrics.starfish_rbc_dag_shadow_comparison_valid.get(), + 0, + "autonomous mode must not claim direct-round comparison" + ); + } + } else if starfish_rbc_dag_shadow { let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR * i64::try_from(committee_size).unwrap(); tokio::time::timeout(timeout, async { @@ -563,6 +749,11 @@ mod smoke_tests { run_commit_test_with_shadow("starfish-rbc", Some("mac"), 1640, true).await; } + #[tokio::test] + async fn starfish_rbc_dag_autonomous_clock_advances_without_owning_consensus() { + run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1700, true, true).await; + } + #[tokio::test] async fn starfish_rbc_single_validator_starts_on_current_thread_runtime() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index ce06da66..4f560e27 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -76,6 +76,13 @@ enum Operation { /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_autonomous_clock: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_heartbeat_interval_ms: Option, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -113,6 +120,13 @@ enum Operation { /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_autonomous_clock: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_heartbeat_interval_ms: Option, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -172,6 +186,13 @@ enum Operation { /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_autonomous_clock: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_heartbeat_interval_ms: Option, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, /// Dissemination mode override: @@ -207,6 +228,8 @@ async fn main() -> Result<()> { consensus: consensus_protocol, block_authentication, starfish_rbc_dag_shadow, + starfish_rbc_dag_autonomous_clock, + starfish_rbc_dag_heartbeat_interval_ms, } => { run( authority, @@ -218,6 +241,8 @@ async fn main() -> Result<()> { consensus_protocol, block_authentication, starfish_rbc_dag_shadow, + starfish_rbc_dag_autonomous_clock, + starfish_rbc_dag_heartbeat_interval_ms, ) .await? } @@ -233,6 +258,8 @@ async fn main() -> Result<()> { consensus: consensus_protocol, block_authentication, starfish_rbc_dag_shadow, + starfish_rbc_dag_autonomous_clock, + starfish_rbc_dag_heartbeat_interval_ms, data_dir, base_ip, storage_backend, @@ -253,6 +280,8 @@ async fn main() -> Result<()> { consensus_protocol, block_authentication, starfish_rbc_dag_shadow, + starfish_rbc_dag_autonomous_clock, + starfish_rbc_dag_heartbeat_interval_ms, data_dir, base_ip, storage_backend, @@ -275,6 +304,8 @@ async fn main() -> Result<()> { consensus: consensus_protocol, block_authentication, starfish_rbc_dag_shadow, + starfish_rbc_dag_autonomous_clock, + starfish_rbc_dag_heartbeat_interval_ms, duration_secs, dissemination_mode, } => { @@ -286,6 +317,10 @@ async fn main() -> Result<()> { node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; + node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; + if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + } if consensus_protocol == "starfish-rbc" { node_parameters.refresh_starfish_rbc_protocol_instance(); } @@ -431,6 +466,8 @@ async fn local_benchmark( }; let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; + let starfish_rbc_dag_autonomous_clock_expected = + node_parameters.starfish_rbc_dag_autonomous_clock; // Create temporary directories for each validator let base_dir = PathBuf::from("local-benchmark"); @@ -503,14 +540,22 @@ async fn local_benchmark( ) .await? }; + let validator_metrics = validator.metrics(); if !is_byzantine { - metrics_of_honest_validators.push(validator.metrics()); + metrics_of_honest_validators.push(Arc::clone(&validator_metrics)); reporters_of_honest_validators.push(validator.reporter()) } // Use the same pattern as the run method let handle = tokio::spawn(async move { let (network_result, _metrics_result) = validator.await_completion().await; + if starfish_rbc_dag_autonomous_clock_expected { + validator_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + } else if starfish_rbc_dag_shadow_expected { + validator_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } network_result }); abort_handles.push(handle.abort_handle()); @@ -520,10 +565,13 @@ async fn local_benchmark( if starfish_rbc_dag_shadow_expected { let ready = tokio::time::timeout(Duration::from_secs(30), async { loop { - if metrics_of_honest_validators - .iter() - .all(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) - { + if metrics_of_honest_validators.iter().all(|metrics| { + if starfish_rbc_dag_autonomous_clock_expected { + metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1 + } else { + metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1 + } + }) { break; } tokio::time::sleep(Duration::from_millis(25)).await; @@ -535,12 +583,24 @@ async fn local_benchmark( abort_handle.abort(); } fs::remove_dir_all(&base_dir)?; + let mode = if starfish_rbc_dag_autonomous_clock_expected { + "autonomous clock" + } else { + "direct-comparison shadow" + }; eyre::bail!( - "Starfish-RBC-DAG shadow did not become ready on every honest validator; benchmark was not started" + "Starfish-RBC-DAG {mode} did not become ready on every honest validator; benchmark was not started" ); } } + let autonomous_clock_baselines = starfish_rbc_dag_autonomous_clock_expected.then(|| { + metrics_of_honest_validators + .iter() + .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) + .collect::>() + }); + // Run for specified duration tokio::select! { _ = tokio::time::sleep(Duration::from_secs(duration_secs)) => { @@ -555,6 +615,8 @@ async fn local_benchmark( duration_secs, committee_size, starfish_rbc_dag_shadow_expected, + starfish_rbc_dag_autonomous_clock_expected, + autonomous_clock_baselines.clone(), ); // Abort all tasks @@ -580,9 +642,11 @@ async fn local_benchmark( duration_secs, committee_size, starfish_rbc_dag_shadow_expected, + starfish_rbc_dag_autonomous_clock_expected, + autonomous_clock_baselines, ); fs::remove_dir_all(base_dir)?; - Ok(()) + eyre::bail!("All validators completed before the requested benchmark duration") } } } @@ -598,6 +662,8 @@ async fn run( consensus_protocol: String, block_authentication: Option, starfish_rbc_dag_shadow: bool, + starfish_rbc_dag_autonomous_clock: bool, + starfish_rbc_dag_heartbeat_interval_ms: Option, ) -> Result<()> { tracing::info!("Starting node {authority}"); @@ -612,6 +678,14 @@ async fn run( if starfish_rbc_dag_shadow { public_config.parameters.starfish_rbc_dag_shadow = true; } + if starfish_rbc_dag_autonomous_clock { + public_config.parameters.starfish_rbc_dag_autonomous_clock = true; + } + if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { + public_config + .parameters + .starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" ))?; @@ -650,6 +724,8 @@ async fn dryrun( consensus_protocol: String, block_authentication: Option, starfish_rbc_dag_shadow: bool, + starfish_rbc_dag_autonomous_clock: bool, + starfish_rbc_dag_heartbeat_interval_ms: Option, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -693,6 +769,10 @@ async fn dryrun( node_parameters.compress_network = compress_network; node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; + node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; + if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + } ensure_starfish_rbc_protocol_instance(&consensus_protocol, &mut node_parameters); if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; @@ -873,6 +953,9 @@ mod tests { "--block-authentication", "mac", "--starfish-rbc-dag-shadow", + "--starfish-rbc-dag-autonomous-clock", + "--starfish-rbc-dag-heartbeat-interval-ms", + "125", ]) .unwrap(); @@ -880,6 +963,8 @@ mod tests { consensus, block_authentication, starfish_rbc_dag_shadow, + starfish_rbc_dag_autonomous_clock, + starfish_rbc_dag_heartbeat_interval_ms, .. } = args.operation else { @@ -888,12 +973,16 @@ mod tests { assert_eq!(consensus, "starfish-rbc"); assert_eq!(block_authentication.as_deref(), Some("mac")); assert!(starfish_rbc_dag_shadow); + assert!(starfish_rbc_dag_autonomous_clock); + assert_eq!(starfish_rbc_dag_heartbeat_interval_ms, Some(125)); } #[test] fn dry_run_starfish_rbc_configuration_gets_a_protocol_instance() { let mut parameters = NodeParameters { starfish_rbc_dag_shadow: true, + starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }; @@ -905,5 +994,7 @@ mod tests { .is_some_and(|instance| instance != [0; 32]) ); assert!(parameters.starfish_rbc_dag_shadow); + assert!(parameters.starfish_rbc_dag_autonomous_clock); + assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 592fff11..8967f99a 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,13 +1,15 @@ # Starfish-RBC-DAG protocol design -Status: milestone-three persisted, non-authoritative shadow runtime; no authoritative protocol, -safety/liveness, or performance claim +Status: milestone-four persisted, non-authoritative optimistic carrier-clock shadow; no +authoritative protocol, safety/liveness, or performance claim The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not -implemented. The current runtime is enabled with `--consensus starfish-rbc ---starfish-rbc-dag-shadow`; it observes the direct prototype without changing its DAG, pacemaker, -commit, or output. The eventual protocol is new, not a transport option or a version-two alias for -`starfish-rbc`. +implemented. The milestone-three direct-header comparison runtime is enabled with `--consensus +starfish-rbc --starfish-rbc-dag-shadow`. Milestone four adds a separate control-only runtime with +`--starfish-rbc-dag-autonomous-clock`; its carrier rounds advance independently through +authenticated admission and empty heartbeats. Both modes leave the direct prototype's DAG, +pacemaker, commit, and output unchanged. The eventual protocol is new, not a transport option or a +version-two alias for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -40,13 +42,23 @@ This is a proposed composition. The reliable-broadcast thresholds are standard, commit rules already exist. Milestone two provides a canonical codec plus deterministic carrier/RBC, certified-projection, decision, and crash-journal models. Milestone three adds an opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired -direct/shadow delivery observations. The direct `starfish-rbc` service remains the only authority: -shadow admission, delivery, recovery, or failure cannot advance a proposal, mark a DAG vertex -clean, vote, commit, or order output. - -Milestone-three restart coverage is deliberately scoped to reopening the shadow actor and its WAL -against an identical recovered direct-header history. It is not a full validator crash-recovery -claim. The authoritative direct `starfish-rbc` baseline does not yet durably record its remote-slot +direct/shadow delivery observations. Milestone four adds an independently authenticated autonomous +heartbeat namespace, sequential `Q`-admitted carrier clock, bounded future buffering, exact-slot +synchronization, and clock-state metrics. The direct `starfish-rbc` service remains the only +authority: shadow admission, delivery, recovery, clock advancement, or failure cannot advance a +proposal, mark a DAG vertex clean, vote, commit, or order output. + +Autonomous mode is intentionally control-only at this milestone. It ignores direct application +headers and uses a distinct WAL and authentication protocol instance. This avoids falsely equating +direct consensus rounds with faster carrier rounds, but also means milestone four does not yet +measure application latency through the new carrier DAG. Application-origin assignment and the +certified consensus projection remain later milestones. + +Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode +requires an identical recovered direct-header history, while autonomous mode reopens its +control-only heartbeat history without direct headers and serves byte-identical exact-slot +responses. This is not a full validator crash-recovery claim. The authoritative direct +`starfish-rbc` baseline does not yet durably record its remote-slot ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest recovered own header dirty. Full validator restart remains fail-stop until those direct-RBC locks @@ -59,7 +71,9 @@ can perturb authoritative timing even though no shadow result is consumed by con one has no feature handshake. Every validator in a shadow run must use a binary that understands the append-only shadow wire variants and the flag must be deployed committee-wide; an older peer will reject an unknown bincode variant and may close the shared connection. Mixed-version or -partially enabled runs are not valid comparisons. +partially enabled runs are not valid comparisons. Mirror and autonomous carriers derive distinct +authentication protocol instances so they cannot cross-admit, but this fail-closed boundary is not +a substitute for capability negotiation. Shadow shutdown is bounded so an observational WAL failure cannot indefinitely block validator shutdown. If that timeout fires, the blocking worker may still hold the shadow WAL's single-writer @@ -764,11 +778,22 @@ newest current-process observation (`<= 4`). These are empirical benchmark cover asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as proof of a protocol failure. -The milestone-three actor reserves the full hard 64-entry queue so several fan-in bursts can wait -behind a synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars -and allocator overhead). It still verifies that one peer fan-in plus five local/control inputs fits; -shadow runs above 60 validators are rejected rather than silently producing an incomplete -comparison. +The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a +synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars and +allocator overhead). Mirror mode budgets one peer fan-in plus five local/control inputs and accepts +at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and +exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs +are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, +healthy proactive rounds receive a repair grace period, and exact synchronization is rate-limited +per peer. Requested historical slots remain recoverable beyond the benchmark-only unsolicited +retention window. + +Autonomous benchmark validity is separate from delivery comparison validity. +`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the WAL and heartbeat counters must progress, +the carrier round and embedded-RBC delivery count must advance during the measured interval, +recovery must drain, and the reported clock-state/backlog and cross-node skew must remain within the +configured empirical guards. These checks establish that the observational carrier plane stayed +live and bounded; they are not a partial-synchrony proof. ## 15. Safety obligations @@ -841,7 +866,12 @@ minimum it must cover: - persisted shadow-actor restart with byte-identical retransmission against an identical recovered direct-header history, bounded overload, poisoned-tag candidate retention, exact recovery, and paired delivery observations against the current direct RBC kernel. Full validator restart is - excluded until the authoritative direct-RBC locks are durable. + excluded until the authoritative direct-RBC locks are durable; and +- autonomous actor progress at `n = 4` and `n = 7`, no steady-state repair polling on healthy + proactive rounds, exact-slot synchronization with idempotent late responses and per-peer rate + limiting, multi-round convergence after a validator falls behind, control-only WAL reopen, + distinct authentication namespace, and an integration check that direct Starfish-RBC continues + committing while the observational carrier clock advances. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -871,6 +901,15 @@ transition and validates through a clone-based reference reducer. Those costs ar charged as protocol overhead: performance runs require incremental state transitions/checkpoints or an equivalently durable baseline, plus separate WAL/fsync accounting. +As an implementation-continuity check, a 10-validator, 60-second local run on 2026-08-11 used the +AWS RTT emulator, a nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. +All validators reached carrier round 196 with zero skew and zero pending recovery; the control +plane recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative direct +Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 end-to-end +latency, and 0.45 MB/s average outbound bandwidth. This is not a comparative performance claim: +the cutoff includes the local generator warmup, and the control-only shadow still performs +per-transition fsync/reference-model work on the authoritative network socket. + ## 19. Contained implementation milestones Every milestone is committed separately. @@ -882,15 +921,17 @@ Every milestone is committed separately. frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable journal models; and deterministic adversarial simulations. No network or existing consensus path changes. -3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the current direct - `starfish-rbc` service, cache the validated committee/domain identity rather than re-hashing all - public keys per carrier, journal ingress and local locks, and compare embedded versus direct RBC - delivery through current-process paired observations. Direct RBC remains authoritative; shadow - results never affect proposals or commits. The reference WAL/reducer is a correctness instrument, - not yet an interpretable protocol-performance path. -4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum - clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses - the current baseline. +3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the + current direct `starfish-rbc` service, cache the validated committee/domain identity rather than + re-hashing all public keys per carrier, journal ingress and local locks, and compare embedded + versus direct RBC delivery through current-process paired observations. Direct RBC remains + authoritative; shadow results never affect proposals or commits. The reference WAL/reducer is a + correctness instrument, not yet an interpretable protocol-performance path. +4. **Optimistic carrier clock (implemented, opt-in control shadow):** run a separately namespaced, + control-only heartbeat carrier plane with the distinct authenticated-admission latch, sequential + quorum clock, bounded future buffer, exact-slot synchronization, durable restart, and clock + validity metrics while consensus still uses the current direct baseline. Application headers are + not assigned to autonomous carrier rounds yet. 5. **Authoritative embedded RBC:** remove direct ECHO/READY authority only after shadow tests show identical delivery under reordering, loss, equivocation, poisoned tags, and restart. 6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit @@ -909,7 +950,8 @@ the executable model or measured prototype: - production maximum future-carrier buffer and payload runahead (the executable model deliberately uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); -- the control-heartbeat rate under low load and backpressure; +- the production control-heartbeat rate under low load and backpressure (the autonomous shadow's + configurable 250 ms default is an empirical test value, not a protocol constant); - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and From d6f11cd29318c98fd4319ac1dff460bbd78abde2 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:22:34 +0200 Subject: [PATCH 32/62] Isolate RBC-DAG benchmark persistence overhead --- README.md | 52 +++- crates/orchestrator/src/benchmark.rs | 7 +- crates/orchestrator/src/main.rs | 12 + crates/orchestrator/src/measurements.rs | 75 +++++- crates/orchestrator/src/protocol/starfish.rs | 8 + crates/starfish-core/src/config.rs | 29 +++ crates/starfish-core/src/metrics.rs | 238 ++++++++++++++---- crates/starfish-core/src/net_sync.rs | 35 ++- .../src/starfish_rbc_dag/storage.rs | 75 +++++- .../src/starfish_rbc_dag_shadow.rs | 23 +- .../src/starfish_rbc_dag_shadow_service.rs | 90 ++++++- crates/starfish-core/src/validator.rs | 61 ++++- crates/starfish/src/main.rs | 62 ++++- docs/starfish-rbc-dag-protocol.md | 69 +++-- 14 files changed, 715 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 8d6c8ded..a0fdb39f 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,11 @@ authoritative in both modes, and shadow failures or results cannot affect propos output as protocol state. Shadow traffic still shares the validator's network socket and bandwidth, so it can perturb timing, and it must be enabled only on a homogeneous new-binary committee; there is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` selector is not -implemented yet. The shadow uses per-transition fsync and a clone-based reference reducer, so it is -a correctness instrument, not a fair performance baseline, and carries no safety or liveness claim. +implemented yet. The default shadow uses per-transition fsync and a clone-based reference reducer, +so it is a correctness instrument, not a fair performance baseline, and carries no safety or +liveness claim. Benchmark runs may add `--starfish-rbc-dag-shadow-buffered-wal`; this preserves the +ordered checksummed log and syncs it on clean shutdown, but deliberately gives up crash safety for +that run. Appended and durably synchronized WAL records are reported separately. Its WAL can reopen the shadow actor, but this is not full validator crash recovery: authoritative direct Starfish-RBC phase and delivery locks are not durable yet, so that baseline remains fail-stop across process restart. The design and proof obligations are documented in the @@ -72,19 +75,31 @@ observational path was disabled or shed work and the comparison must be discarde production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between the cumulative direct and shadow delivery counters. Autonomous runs instead require -`starfish_rbc_dag_shadow_clock_valid == 1`, durable heartbeat progress, advancing carrier rounds, +`starfish_rbc_dag_shadow_clock_valid == 1`, heartbeat/WAL progress, advancing carrier rounds, in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 in autonomous mode. -An exploratory 10-validator, 60-second local run with the AWS RTT emulator, nominal 1,000 tx/s -load, MAC authentication, and a 250 ms autonomous heartbeat completed with a `VALID` clock -verdict on 2026-08-11. All validators reached carrier round 196 with zero skew and zero pending -recovery; the run recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative -direct Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 -end-to-end latency, and 0.45 MB/s average outbound bandwidth. This is a prototype continuity -result, not a fair performance comparison: the 60-second cutoff includes the local generator -warmup, and the control-only shadow still performs per-transition fsync/reference-model work while -sharing the authoritative socket. +A matched 10-validator, 60-second-active-window local A/B on 2026-08-11 used the AWS RTT emulator, +nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. + +| Profile | Verdict | TPS | Block latency | E2E latency | Outbound BW | +|---|---:|---:|---:|---:|---:| +| Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | +| Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | + +Every shadow validator reached carrier round 275 with zero skew and pending recovery; the run +recorded 2,749 heartbeats, 27,180 embedded-RBC deliveries, and 27,424 WAL batches. Thus this +prototype's carrier/RBC work had no measurable latency penalty in this run; the extra shadow +traffic cost about 0.05 MB/s outbound. The result is not yet application latency through RBC-DAG +because direct Starfish-RBC remains authoritative. + +The same 250 ms experiment with the default crash-safe, per-transition-fsync WAL shed shadow work +and ended `INVALID` (9/10 valid validators); its authoritative path slowed to 2,569.1 ms block and +3,067.4 ms end-to-end latency. This isolates synchronous shadow persistence as the prior observer +effect, rather than a carrier-DAG latency regression. Use the buffered profile for protocol +benchmarks and the default profile for crash/replay tests; do not cite an `INVALID` run as a +protocol result. The local harness now starts its timer after transaction-generator warmup, +subtracts warmup counters, and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's @@ -292,6 +307,19 @@ cargo run --release --bin starfish -- local-benchmark \ Additional flags: `--dissemination-mode`, `--adversarial-latency`, `--uniform-latency-ms`. +For the non-authoritative autonomous RBC-DAG benchmark profile: + +```bash +cargo run --release --bin starfish -- local-benchmark \ + --committee-size 10 --load 1000 --consensus starfish-rbc \ + --block-authentication mac --mimic-extra-latency \ + --starfish-rbc-dag-shadow --starfish-rbc-dag-autonomous-clock \ + --starfish-rbc-dag-heartbeat-interval-ms 250 \ + --starfish-rbc-dag-shadow-buffered-wal --duration-secs 60 +``` + +The buffered WAL is benchmark-only and is not crash-safe. + ### Local dryrun with monitoring and dashboard The dryrun script launches a Docker-based local testbed with diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index f29acaf9..7eae00ad 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -116,6 +116,8 @@ pub struct BenchmarkRunSummary { #[serde(default)] pub shadow_delivery_ambiguous: usize, #[serde(default)] + pub shadow_wal_appended_records: usize, + #[serde(default)] pub shadow_wal_durable_records: usize, #[serde(default)] pub shadow_pending_recovery: usize, @@ -173,7 +175,7 @@ impl BenchmarkRunSummary { shadow_comparison_valid_nodes,shadow_direct_deliveries,shadow_deliveries,\ shadow_delivery_matches,\ shadow_delivery_mismatches,shadow_delivery_ambiguous,\ - shadow_wal_durable_records,shadow_pending_recovery,\ + shadow_wal_appended_records,shadow_wal_durable_records,shadow_pending_recovery,\ shadow_unpaired_direct,shadow_unpaired_shadow,\ shadow_unpaired_max_round_lag,\ shadow_autonomous_clock_enabled,shadow_autonomous_clock_valid,\ @@ -225,6 +227,7 @@ impl BenchmarkRunSummary { self.shadow_delivery_matches.to_string(), self.shadow_delivery_mismatches.to_string(), self.shadow_delivery_ambiguous.to_string(), + self.shadow_wal_appended_records.to_string(), self.shadow_wal_durable_records.to_string(), self.shadow_pending_recovery.to_string(), self.shadow_unpaired_direct.to_string(), @@ -853,6 +856,7 @@ pub mod test { #[test] fn benchmark_csv_includes_autonomous_clock_verdict_and_state() { let summary = BenchmarkRunSummary { + shadow_wal_appended_records: 9, shadow_autonomous_clock_enabled: true, shadow_autonomous_clock_valid: true, shadow_autonomous_clock_valid_nodes: 4, @@ -873,6 +877,7 @@ pub mod test { assert_eq!(headers.len(), values.len()); for (header, expected) in [ + ("shadow_wal_appended_records", "9"), ("shadow_autonomous_clock_enabled", "true"), ("shadow_autonomous_clock_valid", "true"), ("shadow_autonomous_clock_valid_nodes", "4"), diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 1c30ea52..b00903d9 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -77,6 +77,11 @@ pub struct Opts { #[clap(long, value_name = "INT", global = true)] starfish_rbc_dag_heartbeat_interval_ms: Option, + /// Benchmark-only: sync the framed shadow WAL only at clean shutdown. + /// This removes per-transition disk pressure and is not crash-safe. + #[clap(long, global = true)] + starfish_rbc_dag_shadow_buffered_wal: bool, + /// The type of operation to run. #[clap(subcommand)] operation: Operation, @@ -87,6 +92,7 @@ struct StarfishRbcDagOverrides { shadow: bool, autonomous_clock: bool, heartbeat_interval_ms: Option, + buffered_wal: bool, } /// The type of operation to run. @@ -893,6 +899,9 @@ fn load_benchmark_configs( if let Some(interval_ms) = starfish_rbc_dag.heartbeat_interval_ms { node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; } + if starfish_rbc_dag.buffered_wal { + node_parameters.starfish_rbc_dag_shadow_buffered_wal = true; + } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1077,6 +1086,7 @@ async fn run( shadow: opts.starfish_rbc_dag_shadow, autonomous_clock: opts.starfish_rbc_dag_autonomous_clock, heartbeat_interval_ms: opts.starfish_rbc_dag_heartbeat_interval_ms, + buffered_wal: opts.starfish_rbc_dag_shadow_buffered_wal, }; match opts.operation { Operation::Testbed { action } => match action { @@ -2375,6 +2385,7 @@ mod tests { "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-heartbeat-interval-ms", "125", + "--starfish-rbc-dag-shadow-buffered-wal", "--protocols", "starfish-rbc", ]) @@ -2383,6 +2394,7 @@ mod tests { assert_eq!(opts.block_authentication.as_deref(), Some("mac")); assert!(opts.starfish_rbc_dag_shadow); assert!(opts.starfish_rbc_dag_autonomous_clock); + assert!(opts.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(opts.starfish_rbc_dag_heartbeat_interval_ms, Some(125)); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 945900ab..e63c0d0f 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -286,7 +286,9 @@ impl Measurement { } x if matches!( x.as_str(), - "starfish_rbc_dag_shadow_wal_durable_batches_total" + "starfish_rbc_dag_shadow_wal_appended_batches_total" + | "starfish_rbc_dag_shadow_wal_appended_records_total" + | "starfish_rbc_dag_shadow_wal_durable_batches_total" | "starfish_rbc_dag_shadow_wal_durable_records_total" | "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total" ) => @@ -1022,6 +1024,8 @@ impl MeasurementsCollection { .sum_count_bucket_increments("starfish_rbc_dag_shadow_inputs_total", "delivery,shadow"); let shadow_wal_durable_records = self.sum_scalar_counter_increments("starfish_rbc_dag_shadow_wal_durable_records_total"); + let shadow_wal_appended_records = self + .sum_scalar_counter_increments("starfish_rbc_dag_shadow_wal_appended_records_total"); let shadow_pending_recovery = self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_pending_recovery"); let shadow_unpaired_direct = @@ -1068,7 +1072,7 @@ impl MeasurementsCollection { *scraper_id, "delivery,shadow", ) && self.scalar_counter_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_wal_durable_records_total", + "starfish_rbc_dag_shadow_wal_appended_records_total", *scraper_id, ) && self.latest_scalar_equals( "starfish_rbc_dag_shadow_pending_recovery", @@ -1150,10 +1154,10 @@ impl MeasurementsCollection { *scraper_id, "delivery,shadow", ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_durable_batches_total", + "starfish_rbc_dag_shadow_wal_appended_batches_total", *scraper_id, ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_durable_records_total", + "starfish_rbc_dag_shadow_wal_appended_records_total", *scraper_id, ) && self.scalar_gauge_increased( "starfish_rbc_dag_shadow_carrier_round", @@ -1232,6 +1236,7 @@ impl MeasurementsCollection { shadow_delivery_matches, shadow_delivery_mismatches, shadow_delivery_ambiguous, + shadow_wal_appended_records, shadow_wal_durable_records, shadow_pending_recovery, shadow_unpaired_direct, @@ -1300,7 +1305,7 @@ impl MeasurementsCollection { b->"RBC-DAG shadow:", format!( "valid={} ({}/{} validators), direct={}, shadow={}, matches={}, \ - mismatches={}, ambiguous={}, WAL records={}, pending recovery={}, \ + mismatches={}, ambiguous={}, WAL appended/durable records={}/{}, pending recovery={}, \ unpaired direct/shadow={}/{}, max unpaired lag={} rounds", summary.shadow_comparison_valid, summary.shadow_comparison_valid_nodes, @@ -1310,6 +1315,7 @@ impl MeasurementsCollection { summary.shadow_delivery_matches, summary.shadow_delivery_mismatches, summary.shadow_delivery_ambiguous, + summary.shadow_wal_appended_records, summary.shadow_wal_durable_records, summary.shadow_pending_recovery, summary.shadow_unpaired_direct, @@ -1323,7 +1329,7 @@ impl MeasurementsCollection { b->"RBC-DAG autonomous clock:", format!( "valid={} ({}/{} validators), carrier rounds={}..={}, embedded deliveries={}, phase backlog={}, \ - admitted authors/stake min={}/{}, buffered authenticated={}, WAL records={}, \ + admitted authors/stake min={}/{}, buffered authenticated={}, WAL appended/durable records={}/{}, \ pending recovery={}", summary.shadow_autonomous_clock_valid, summary.shadow_autonomous_clock_valid_nodes, @@ -1335,6 +1341,7 @@ impl MeasurementsCollection { summary.shadow_autonomous_clock_admitted_authors_min, summary.shadow_autonomous_clock_admitted_stake_min, summary.shadow_autonomous_clock_buffered_authenticated_total, + summary.shadow_wal_appended_records, summary.shadow_wal_durable_records, summary.shadow_pending_recovery, ) @@ -1479,6 +1486,15 @@ mod test { ..Measurement::default() }, ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_wal_appended_records_total".to_owned(), + Measurement { + count: wal_durable_records, + scalar: wal_durable_records as f64, + ..Measurement::default() + }, + ); collection.add( scraper_id, "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), @@ -1581,6 +1597,26 @@ mod test { ..Measurement::default() }, ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_wal_appended_batches_total".to_owned(), + Measurement { + timestamp, + count: wal_durable_records, + scalar: wal_durable_records as f64, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_shadow_wal_appended_records_total".to_owned(), + Measurement { + timestamp, + count: wal_durable_records, + scalar: wal_durable_records as f64, + ..Measurement::default() + }, + ); collection.add( scraper_id, "starfish_rbc_dag_shadow_wal_durable_batches_total".to_owned(), @@ -1774,6 +1810,8 @@ starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="ambigu # TYPE starfish_rbc_dag_shadow_inputs_total counter starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="shadow"} 7 starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="direct"} 7 +# TYPE starfish_rbc_dag_shadow_wal_appended_records_total counter +starfish_rbc_dag_shadow_wal_appended_records_total{node="node-0"} 42 # TYPE starfish_rbc_dag_shadow_wal_durable_records_total counter starfish_rbc_dag_shadow_wal_durable_records_total{node="node-0"} 42 # TYPE starfish_rbc_dag_shadow_wal_replayed_batches gauge @@ -1801,6 +1839,10 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 measurements["starfish_rbc_dag_shadow_inputs_total"].count_buckets["delivery,shadow"], 7 ); + assert_eq!( + measurements["starfish_rbc_dag_shadow_wal_appended_records_total"].scalar, + 42.0 + ); assert_eq!( measurements["starfish_rbc_dag_shadow_wal_durable_records_total"].scalar, 42.0 @@ -1825,6 +1867,7 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 assert_eq!(summary.shadow_direct_deliveries, 7); assert_eq!(summary.shadow_deliveries, 7); assert_eq!(summary.shadow_delivery_matches, 7); + assert_eq!(summary.shadow_wal_appended_records, 42); assert_eq!(summary.shadow_wal_durable_records, 42); assert_eq!(summary.shadow_unpaired_direct, 2); assert_eq!(summary.shadow_unpaired_shadow, 1); @@ -1896,6 +1939,24 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert!(!summary.shadow_autonomous_clock_valid); } + #[test] + fn autonomous_clock_buffered_wal_uses_appended_progress_without_claiming_durability() { + let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 10, 0, 1, 1, 0, 5); + collection + .data + .remove("starfish_rbc_dag_shadow_wal_durable_batches_total"); + collection + .data + .remove("starfish_rbc_dag_shadow_wal_durable_records_total"); + + let summary = collection.benchmark_run_summary(); + assert!(summary.shadow_autonomous_clock_valid); + assert_eq!(summary.shadow_wal_appended_records, 5); + assert_eq!(summary.shadow_wal_durable_records, 0); + } + #[test] fn missing_final_autonomous_scrape_invalidates_only_clock_verdict() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); @@ -1954,7 +2015,7 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 } #[test] - fn autonomous_clock_valid_gauge_without_durable_progress_is_invalid() { + fn autonomous_clock_valid_gauge_without_wal_progress_is_invalid() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 8b3bc56c..1bb38995 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -324,6 +324,7 @@ impl StarfishProtocol { // validator configuration. node_parameters.starfish_rbc_dag_shadow = false; node_parameters.starfish_rbc_dag_autonomous_clock = false; + node_parameters.starfish_rbc_dag_shadow_buffered_wal = false; node_parameters.starfish_rbc_dag_heartbeat_interval_ms = config::node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(); } @@ -370,6 +371,7 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, starfish_rbc_dag_heartbeat_interval_ms: 125, + starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); let parameters = @@ -381,6 +383,7 @@ mod tests { ); assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); + assert!(parameters.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } @@ -390,6 +393,7 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, starfish_rbc_dag_heartbeat_interval_ms: 125, + starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); let parameters = @@ -403,6 +407,10 @@ mod tests { !parameters.starfish_rbc_dag_autonomous_clock, "a global autonomous-clock flag must not leak into non-RBC comparison members" ); + assert!( + !parameters.starfish_rbc_dag_shadow_buffered_wal, + "a global buffered-WAL flag must not leak into non-RBC comparison members" + ); assert_eq!( parameters.starfish_rbc_dag_heartbeat_interval_ms, node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 12c3a161..66f0a305 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -81,6 +81,12 @@ pub struct NodeParameters { /// carriers. The value is ignored unless the autonomous clock is enabled. #[serde(default = "node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms")] pub starfish_rbc_dag_heartbeat_interval_ms: u64, + /// Benchmark-only profile that writes the framed shadow WAL in order but + /// calls `sync_all` only during clean shutdown. This removes persistence + /// pressure from latency experiments and deliberately forfeits the + /// shadow's crash-safety claim for that run. + #[serde(default)] + pub starfish_rbc_dag_shadow_buffered_wal: bool, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -164,6 +170,7 @@ impl Default for NodeParameters { starfish_rbc_dag_autonomous_clock: false, starfish_rbc_dag_heartbeat_interval_ms: node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), + starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), @@ -406,6 +413,16 @@ impl NodePrivateConfig { self.storage_path .join("starfish-rbc-dag-autonomous-clock-v1.wal") } + + pub fn starfish_rbc_dag_shadow_buffered_benchmark_wal(&self) -> PathBuf { + self.storage_path + .join("starfish-rbc-dag-shadow-buffered-benchmark-v1.wal") + } + + pub fn starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal(&self) -> PathBuf { + self.storage_path + .join("starfish-rbc-dag-autonomous-clock-buffered-benchmark-v1.wal") + } } impl ImportExport for NodePrivateConfig {} @@ -422,6 +439,7 @@ mod tests { assert_eq!(parameters.starfish_rbc_protocol_instance, None); assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); + assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); assert_eq!( parameters.starfish_rbc_dag_heartbeat_interval_ms, node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() @@ -438,6 +456,7 @@ mod tests { ); assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); + assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!( decoded.starfish_rbc_dag_heartbeat_interval_ms, node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() @@ -450,6 +469,7 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, starfish_rbc_dag_heartbeat_interval_ms: 125, + starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -457,6 +477,7 @@ mod tests { let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); assert!(decoded.starfish_rbc_dag_shadow); assert!(decoded.starfish_rbc_dag_autonomous_clock); + assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(decoded.starfish_rbc_dag_heartbeat_interval_ms, 125); } @@ -469,6 +490,14 @@ mod tests { private_config.starfish_rbc_dag_shadow_wal(), private_config.starfish_rbc_dag_autonomous_clock_wal() ); + assert_ne!( + private_config.starfish_rbc_dag_autonomous_clock_wal(), + private_config.starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal() + ); + assert_ne!( + private_config.starfish_rbc_dag_shadow_buffered_benchmark_wal(), + private_config.starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal() + ); assert_eq!( private_config.starfish_rbc_dag_autonomous_clock_wal(), Path::new("benchmark") diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index cbb56c7a..a00075c0 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -51,6 +51,32 @@ pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG: i64 = 4; pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR: i64 = 16; pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR: i64 = 2; +const LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES: &[&str] = &[ + "subscribe_broadcast", + "batch", + "missing_parents", + "missing_tx_data", + "partial_sig", + "cert_echo", + "cert_vote", + "cert_ready", + "cert_batch", + "sailfish_timeout", + "sailfish_no_vote", + "unprovable_cert_request", + "round_gap_request", + "rbc_initial", + "rbc_echo", + "rbc_ready", + "rbc_header_request", + "rbc_header_response", + "rbc_dag_shadow_carrier", + "rbc_dag_shadow_carrier_request", + "rbc_dag_shadow_carrier_response", + "rbc_dag_shadow_carrier_sync_request", + "rbc_dag_shadow_carrier_sync_response", +]; + #[derive(Clone)] pub struct Metrics { pub benchmark_duration: IntCounter, @@ -155,6 +181,8 @@ pub struct Metrics { // consensus state. pub starfish_rbc_dag_shadow_inputs_total: IntCounterVec, pub starfish_rbc_dag_shadow_delivery_comparisons_total: IntCounterVec, + pub starfish_rbc_dag_shadow_wal_appended_batches_total: IntCounter, + pub starfish_rbc_dag_shadow_wal_appended_records_total: IntCounter, pub starfish_rbc_dag_shadow_wal_durable_batches_total: IntCounter, pub starfish_rbc_dag_shadow_wal_durable_records_total: IntCounter, pub starfish_rbc_dag_shadow_wal_replayed_batches: IntGauge, @@ -244,6 +272,18 @@ pub struct AutonomousClockBenchmarkBaseline { carrier_round: i64, } +/// Per-validator cumulative counters sampled at the exact start of a local +/// benchmark's active transaction window. Rates subtract this snapshot so +/// connection warmup and shadow-WAL replay are not charged to the protocol. +#[derive(Clone, Debug, Default)] +pub struct LocalBenchmarkCounterBaseline { + sequenced_transactions: u64, + dag_state_entries: u64, + bytes_sent: u64, + bytes_received: u64, + outbound_messages: Vec<(u64, u64)>, +} + #[derive(Debug, Eq, PartialEq)] struct AutonomousClockBenchmarkSummary { valid_nodes: usize, @@ -299,11 +339,11 @@ fn summarize_autonomous_clock_benchmark( .get() > baseline.delivered_carriers && metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total + .starfish_rbc_dag_shadow_wal_appended_batches_total .get() > baseline.wal_batches && metrics - .starfish_rbc_dag_shadow_wal_durable_records_total + .starfish_rbc_dag_shadow_wal_appended_records_total .get() > baseline.wal_records && metrics.starfish_rbc_dag_shadow_carrier_round.get() > baseline.carrier_round @@ -349,7 +389,7 @@ fn summarize_autonomous_clock_benchmark( .iter() .map(|metrics| { metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total + .starfish_rbc_dag_shadow_wal_appended_batches_total .get() }) .sum(); @@ -357,7 +397,7 @@ fn summarize_autonomous_clock_benchmark( .iter() .map(|metrics| { metrics - .starfish_rbc_dag_shadow_wal_durable_records_total + .starfish_rbc_dag_shadow_wal_appended_records_total .get() }) .sum(); @@ -447,12 +487,38 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .get(), - wal_batches: self.starfish_rbc_dag_shadow_wal_durable_batches_total.get(), - wal_records: self.starfish_rbc_dag_shadow_wal_durable_records_total.get(), + wal_batches: self + .starfish_rbc_dag_shadow_wal_appended_batches_total + .get(), + wal_records: self + .starfish_rbc_dag_shadow_wal_appended_records_total + .get(), carrier_round: self.starfish_rbc_dag_shadow_carrier_round.get(), } } + pub fn local_benchmark_counter_baseline(&self) -> LocalBenchmarkCounterBaseline { + LocalBenchmarkCounterBaseline { + sequenced_transactions: self.sequenced_transactions_total.get(), + dag_state_entries: self.dag_state_entries.get(), + bytes_sent: self.bytes_sent_total.get(), + bytes_received: self.bytes_received_total.get(), + outbound_messages: LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES + .iter() + .map(|request_type| { + ( + self.network_message_bytes_sent_total + .with_label_values(&[request_type]) + .get(), + self.network_requests_sent_total + .with_label_values(&[request_type]) + .get(), + ) + }) + .collect(), + } + } + pub fn new( registry: &Registry, committee: Option<&Committee>, @@ -776,15 +842,29 @@ impl Metrics { registry, ) .unwrap(), + starfish_rbc_dag_shadow_wal_appended_batches_total: + register_int_counter_with_registry!( + "starfish_rbc_dag_shadow_wal_appended_batches_total", + "Starfish-RBC-DAG shadow WAL batches appended to the framed log", + registry, + ) + .unwrap(), + starfish_rbc_dag_shadow_wal_appended_records_total: + register_int_counter_with_registry!( + "starfish_rbc_dag_shadow_wal_appended_records_total", + "Starfish-RBC-DAG shadow WAL records appended to the framed log", + registry, + ) + .unwrap(), starfish_rbc_dag_shadow_wal_durable_batches_total: register_int_counter_with_registry!( "starfish_rbc_dag_shadow_wal_durable_batches_total", - "Starfish-RBC-DAG shadow WAL batches durably synchronized", + "Starfish-RBC-DAG shadow WAL batches synchronized before event exposure", registry, ) .unwrap(), starfish_rbc_dag_shadow_wal_durable_records_total: register_int_counter_with_registry!( "starfish_rbc_dag_shadow_wal_durable_records_total", - "Starfish-RBC-DAG shadow WAL records durably synchronized", + "Starfish-RBC-DAG shadow WAL records synchronized before event exposure", registry, ) .unwrap(), @@ -1327,32 +1407,69 @@ impl Metrics { starfish_rbc_dag_shadow_expected: bool, starfish_rbc_dag_autonomous_clock_expected: bool, autonomous_clock_baselines: Option>, + counter_baselines: Option>, ) { let num_validators = metrics.len() as u64; // Calculate overall statistics let average_transactions: u64 = metrics .iter() - .map(|m| m.sequenced_transactions_total.get()) + .enumerate() + .map(|(index, metrics)| { + metrics.sequenced_transactions_total.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.sequenced_transactions) + .unwrap_or_default(), + ) + }) .sum::() / num_validators; let average_tps = average_transactions as f64 / duration_secs as f64; let average_blocks_submitted = metrics .iter() - .map(|m| m.dag_state_entries.get()) + .enumerate() + .map(|(index, metrics)| { + metrics.dag_state_entries.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.dag_state_entries) + .unwrap_or_default(), + ) + }) .sum::() / num_validators; let average_bps = average_blocks_submitted as f64 / duration_secs as f64; let average_bytes_sent: u64 = metrics .iter() - .map(|m| m.bytes_sent_total.get()) + .enumerate() + .map(|(index, metrics)| { + metrics.bytes_sent_total.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.bytes_sent) + .unwrap_or_default(), + ) + }) .sum::() / num_validators; let average_bytes_received: u64 = metrics .iter() - .map(|m| m.bytes_received_total.get()) + .enumerate() + .map(|(index, metrics)| { + metrics.bytes_received_total.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.bytes_received) + .unwrap_or_default(), + ) + }) .sum::() / num_validators; let average_reconstructed_sent_to_core: u64 = metrics @@ -1396,6 +1513,21 @@ impl Metrics { .sum::() / num_validators as i64; + // The periodic reporter drains every ten seconds. Pull the final tail + // synchronously so a benchmark cutoff never drops its last samples. + for reporter in &reporters { + reporter + .block_committed_latency + .lock() + .histogram + .receive_all(); + reporter + .transaction_committed_latency + .lock() + .histogram + .receive_all(); + } + let p50_block_committed_latency = reporters .iter() .filter_map(|r| r.block_committed_latency.lock().histogram.pcts([500])) @@ -1445,41 +1577,25 @@ impl Metrics { b->"Average bandwidth in:", format!("{:.2} MB/s", bw_in) ]); - const NETWORK_MESSAGE_TYPES: &[&str] = &[ - "subscribe_broadcast", - "batch", - "missing_parents", - "missing_tx_data", - "partial_sig", - "cert_echo", - "cert_vote", - "cert_ready", - "cert_batch", - "sailfish_timeout", - "sailfish_no_vote", - "unprovable_cert_request", - "round_gap_request", - "rbc_initial", - "rbc_echo", - "rbc_ready", - "rbc_header_request", - "rbc_header_response", - "rbc_dag_shadow_carrier", - "rbc_dag_shadow_carrier_request", - "rbc_dag_shadow_carrier_response", - "rbc_dag_shadow_carrier_sync_request", - "rbc_dag_shadow_carrier_sync_response", - ]; - let outbound_message_breakdown = NETWORK_MESSAGE_TYPES + let outbound_message_breakdown = LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES .iter() - .filter_map(|request_type| { + .enumerate() + .filter_map(|(message_index, request_type)| { let average_bytes = metrics .iter() - .map(|metrics| { - metrics + .enumerate() + .map(|(validator_index, metrics)| { + let current = metrics .network_message_bytes_sent_total .with_label_values(&[request_type]) - .get() + .get(); + let baseline = counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(validator_index)) + .and_then(|baseline| baseline.outbound_messages.get(message_index)) + .map(|(bytes, _)| *bytes) + .unwrap_or_default(); + current.saturating_sub(baseline) }) .sum::() as f64 / num_validators as f64; @@ -1488,11 +1604,19 @@ impl Metrics { } let average_requests = metrics .iter() - .map(|metrics| { - metrics + .enumerate() + .map(|(validator_index, metrics)| { + let current = metrics .network_requests_sent_total .with_label_values(&[request_type]) - .get() + .get(); + let baseline = counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(validator_index)) + .and_then(|baseline| baseline.outbound_messages.get(message_index)) + .map(|(_, requests)| *requests) + .unwrap_or_default(); + current.saturating_sub(baseline) }) .sum::() as f64 / num_validators as f64; @@ -1555,7 +1679,7 @@ impl Metrics { ) ]); table.add_row(row![ - b->"Durable clock progress:", + b->"Clock/WAL progress:", format!( "heartbeats={}, RBC deliveries={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, @@ -2045,6 +2169,12 @@ mod tests { .starfish_rbc_dag_shadow_delivery_comparisons_total .with_label_values(&["match"]) .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_records_total + .inc_by(3); metrics .starfish_rbc_dag_shadow_wal_durable_batches_total .inc(); @@ -2075,6 +2205,8 @@ mod tests { for name in [ "starfish_rbc_dag_shadow_inputs_total", "starfish_rbc_dag_shadow_delivery_comparisons_total", + "starfish_rbc_dag_shadow_wal_appended_batches_total", + "starfish_rbc_dag_shadow_wal_appended_records_total", "starfish_rbc_dag_shadow_wal_durable_batches_total", "starfish_rbc_dag_shadow_wal_durable_records_total", "starfish_rbc_dag_shadow_wal_replayed_batches", @@ -2114,6 +2246,12 @@ mod tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_records_total + .inc_by(2); metrics .starfish_rbc_dag_shadow_wal_durable_batches_total .inc(); @@ -2133,7 +2271,7 @@ mod tests { } #[test] - fn autonomous_clock_summary_requires_every_node_to_make_bounded_durable_progress() { + fn autonomous_clock_summary_requires_every_node_to_make_bounded_wal_progress() { let metrics = vec![ autonomous_clock_metrics(8, 3, 1), autonomous_clock_metrics(9, 4, 2), @@ -2177,6 +2315,12 @@ mod tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_records_total + .inc(); metrics .starfish_rbc_dag_shadow_wal_durable_batches_total .inc(); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index e1089edd..2479ab54 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -46,7 +46,10 @@ use crate::{ }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, - starfish_rbc_dag::{RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId}, + starfish_rbc_dag::{ + RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, + storage::ShadowWalSyncPolicyV1, + }, starfish_rbc_dag_shadow::{ ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, }, @@ -1933,6 +1936,11 @@ impl NetworkSyncer metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } let started = if node_parameters.starfish_rbc_dag_autonomous_clock { + let wal_sync_policy = if node_parameters.starfish_rbc_dag_shadow_buffered_wal { + ShadowWalSyncPolicyV1::OnShutdown + } else { + ShadowWalSyncPolicyV1::EveryBatch + }; start_starfish_rbc_dag_autonomous_clock_service_v1( starfish_rbc_dag_shadow_wal, committee_context, @@ -1942,8 +1950,14 @@ impl NetworkSyncer Duration::from_millis( node_parameters.starfish_rbc_dag_heartbeat_interval_ms, ), + wal_sync_policy, ) } else { + let wal_sync_policy = if node_parameters.starfish_rbc_dag_shadow_buffered_wal { + ShadowWalSyncPolicyV1::OnShutdown + } else { + ShadowWalSyncPolicyV1::EveryBatch + }; start_starfish_rbc_dag_shadow_service_v1( starfish_rbc_dag_shadow_wal, committee_context, @@ -1951,6 +1965,7 @@ impl NetworkSyncer context, authorizer, recovered_local_headers, + wal_sync_policy, ) }; match started { @@ -2283,13 +2298,25 @@ impl NetworkSyncer .with_label_values(&[kind, outcome]) .inc(); } - ShadowServiceEventV1::WalDurable { batches, records } => { + ShadowServiceEventV1::WalAppended { + batches, + records, + durable, + } => { shadow_metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total + .starfish_rbc_dag_shadow_wal_appended_batches_total .inc_by(batches); shadow_metrics - .starfish_rbc_dag_shadow_wal_durable_records_total + .starfish_rbc_dag_shadow_wal_appended_records_total .inc_by(records); + if durable { + shadow_metrics + .starfish_rbc_dag_shadow_wal_durable_batches_total + .inc_by(batches); + shadow_metrics + .starfish_rbc_dag_shadow_wal_durable_records_total + .inc_by(records); + } } ShadowServiceEventV1::Ready { autonomous_clock } => { let verdict = if autonomous_clock { diff --git a/crates/starfish-core/src/starfish_rbc_dag/storage.rs b/crates/starfish-core/src/starfish_rbc_dag/storage.rs index d553658a..8a1ef827 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/storage.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/storage.rs @@ -6,9 +6,11 @@ //! The storage layer deliberately does not encode [`super::journal::JournalEventV1`]. //! A later integration layer owns that versioned codec and the recovery of //! opaque authentication capabilities. This module supplies the durability -//! boundary underneath it: one batch is one checksummed frame, and a caller -//! may expose the corresponding effects only after [`ShadowWalV1::append_batch`] -//! returns successfully. +//! boundary underneath it: one batch is one checksummed frame. The default +//! [`ShadowWalSyncPolicyV1::EveryBatch`] policy allows a caller to expose the +//! corresponding effects only after [`ShadowWalV1::append_batch`] returns. +//! The explicit benchmark-only `OnShutdown` policy preserves ordered replay +//! on a clean stop but does not provide that crash-durability boundary. //! //! Recovery discards only a physically short final frame. A fully present //! frame with a bad header, commit marker, or checksum is reported as @@ -258,6 +260,18 @@ impl From for ShadowWalErrorV1 { } } +/// Persistence boundary used by the non-authoritative shadow WAL. +/// +/// `EveryBatch` is the proof-facing crash-safe mode. `OnShutdown` is an +/// explicit benchmark profile: frames are still written and checksummed in +/// order, but effects may become visible before the kernel has forced those +/// bytes to stable storage. It must never be used to claim crash safety. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ShadowWalSyncPolicyV1 { + EveryBatch, + OnShutdown, +} + pub struct ShadowWalV1 { path: PathBuf, file: File, @@ -265,6 +279,8 @@ pub struct ShadowWalV1 { durable_file_len: u64, batch_count: u64, record_count: u64, + sync_policy: ShadowWalSyncPolicyV1, + batch_sync_count: u64, poisoned: bool, } @@ -278,6 +294,14 @@ impl ShadowWalV1 { pub fn open( path: impl AsRef, namespace: ShadowWalNamespaceV1, + ) -> Result<(Self, ShadowWalRecoveryV1), ShadowWalErrorV1> { + Self::open_with_sync_policy(path, namespace, ShadowWalSyncPolicyV1::EveryBatch) + } + + pub(crate) fn open_with_sync_policy( + path: impl AsRef, + namespace: ShadowWalNamespaceV1, + sync_policy: ShadowWalSyncPolicyV1, ) -> Result<(Self, ShadowWalRecoveryV1), ShadowWalErrorV1> { let path = path.as_ref().to_path_buf(); if let Some(parent) = nonempty_parent(&path) { @@ -320,6 +344,8 @@ impl ShadowWalV1 { durable_file_len: recovery.durable_file_len, batch_count: recovery.batch_count(), record_count: recovery.record_count, + sync_policy, + batch_sync_count: 0, poisoned: false, }; Ok((wal, recovery)) @@ -349,7 +375,13 @@ impl ShadowWalV1 { self.poisoned } - /// Append and fsync one atomic record batch before returning. + #[cfg(test)] + fn batch_sync_count(&self) -> u64 { + self.batch_sync_count + } + + /// Append one atomic record batch, forcing it to stable storage before + /// returning only under [`ShadowWalSyncPolicyV1::EveryBatch`]. /// /// Any seek, write, or fsync failure poisons this handle because the commit /// result may be ambiguous. Drop it and reopen the WAL; recovery will @@ -390,15 +422,21 @@ impl ShadowWalV1 { .checked_add(frame_len) .ok_or(ShadowWalErrorV1::LengthOverflow)?; - if let Err(error) = self + let result = self .file .seek(SeekFrom::Start(start_offset)) .and_then(|_| self.file.write_all(&frame)) - .and_then(|_| self.file.sync_all()) - { + .and_then(|_| match self.sync_policy { + ShadowWalSyncPolicyV1::EveryBatch => self.file.sync_all(), + ShadowWalSyncPolicyV1::OnShutdown => Ok(()), + }); + if let Err(error) = result { self.poisoned = true; return Err(ShadowWalErrorV1::Io(error)); } + if self.sync_policy == ShadowWalSyncPolicyV1::EveryBatch { + self.batch_sync_count = self.batch_sync_count.saturating_add(1); + } self.durable_file_len = end_offset; self.batch_count = next_batch_count; @@ -1362,6 +1400,29 @@ mod tests { wal.shutdown().unwrap(); } + #[test] + fn buffered_benchmark_policy_skips_per_batch_sync_and_reopens_cleanly() { + let directory = tempfile::tempdir().unwrap(); + let path = wal_path(&directory); + let namespace = namespace(0xB1, 0); + let (mut buffered, _) = + ShadowWalV1::open_with_sync_policy(&path, namespace, ShadowWalSyncPolicyV1::OnShutdown) + .unwrap(); + buffered.append_batch(&[b"one".to_vec()]).unwrap(); + buffered.append_batch(&[b"two".to_vec()]).unwrap(); + assert_eq!(buffered.batch_sync_count(), 0); + buffered.shutdown().unwrap(); + + let (mut durable, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); + assert_eq!( + recovery.records(), + vec![b"one".as_slice(), b"two".as_slice()] + ); + durable.append_batch(&[b"three".to_vec()]).unwrap(); + assert_eq!(durable.batch_sync_count(), 1); + durable.shutdown().unwrap(); + } + #[test] fn external_file_mutation_poisoning_requires_reopen() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 8d62a392..1fcfd07d 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -29,7 +29,7 @@ use crate::{ model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, storage::{ MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, - ShadowWalSummaryV1, ShadowWalV1, + ShadowWalSummaryV1, ShadowWalSyncPolicyV1, ShadowWalV1, }, }, types::{ @@ -489,16 +489,35 @@ pub(crate) struct StarfishRbcDagShadowV1 { } impl StarfishRbcDagShadowV1 { + #[cfg(test)] pub(crate) fn open( path: impl AsRef, committee: RbcDagCommitteeContextV1, own_authority: AuthorityIndex, context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, + ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { + Self::open_with_wal_sync_policy( + path, + committee, + own_authority, + context, + authorizer, + ShadowWalSyncPolicyV1::EveryBatch, + ) + } + + pub(crate) fn open_with_wal_sync_policy( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { validate_configuration(&committee, own_authority, context, &authorizer)?; let namespace = ShadowWalNamespaceV1::new(context, own_authority); - let (wal, recovery) = ShadowWalV1::open(path, namespace)?; + let (wal, recovery) = ShadowWalV1::open_with_sync_policy(path, namespace, wal_sync_policy)?; let replayed_batches = recovery.batch_count(); let discarded_tail_bytes = recovery.discarded_tail_bytes(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 27787191..62501be8 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -35,6 +35,7 @@ use crate::{ starfish_rbc_dag::{ MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, model::{ModelEffect, ModelError}, + storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, @@ -376,9 +377,10 @@ pub(crate) enum ShadowServiceEventV1 { kind: &'static str, outcome: &'static str, }, - WalDurable { + WalAppended { batches: u64, records: u64, + durable: bool, }, Recovered { batches: u64, @@ -564,6 +566,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, + wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -580,6 +583,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::DirectMirror, + wal_sync_policy, ) } @@ -590,6 +594,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, heartbeat_interval: Duration, + wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -609,6 +614,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( authorizer, Vec::new(), ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + wal_sync_policy, ) } @@ -620,6 +626,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, mode: ShadowServiceModeV1, + wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -718,7 +725,14 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); let task = tokio::spawn(async move { let opened = tokio::task::spawn_blocking(move || { - StarfishRbcDagShadowV1::open(path, committee, own_authority, context, authorizer) + StarfishRbcDagShadowV1::open_with_wal_sync_policy( + path, + committee, + own_authority, + context, + authorizer, + wal_sync_policy, + ) }) .await; let (core, open_report) = match opened { @@ -843,6 +857,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let state = ShadowServiceStateV1 { core, mode, + wal_sync_policy, own_authority, committee_size, events: event_tx, @@ -968,6 +983,7 @@ impl ShadowComparisonBacklogV1 { struct ShadowServiceStateV1 { core: StarfishRbcDagShadowV1, mode: ShadowServiceModeV1, + wal_sync_policy: ShadowWalSyncPolicyV1, own_authority: AuthorityIndex, committee_size: usize, events: mpsc::Sender, @@ -1156,7 +1172,11 @@ impl ShadowServiceStateV1 { let batches = after.0.saturating_sub(before.0); let records = after.1.saturating_sub(before.1); if batches != 0 || records != 0 { - self.emit(ShadowServiceEventV1::WalDurable { batches, records }); + self.emit(ShadowServiceEventV1::WalAppended { + batches, + records, + durable: self.wal_sync_policy == ShadowWalSyncPolicyV1::EveryBatch, + }); } } @@ -2045,6 +2065,7 @@ mod tests { self.context, ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), recovered, + ShadowWalSyncPolicyV1::EveryBatch, ) .unwrap() } @@ -2068,6 +2089,23 @@ mod tests { StarfishRbcDagShadowServiceHandleV1, mpsc::Receiver, JoinHandle<()>, + ) { + self.start_autonomous_with_policy( + authority, + heartbeat_interval, + ShadowWalSyncPolicyV1::EveryBatch, + ) + } + + fn start_autonomous_with_policy( + &self, + authority: AuthorityIndex, + heartbeat_interval: Duration, + wal_sync_policy: ShadowWalSyncPolicyV1, + ) -> ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, ) { start_starfish_rbc_dag_autonomous_clock_service_v1( &self.paths[authority as usize], @@ -2076,6 +2114,7 @@ mod tests { self.context, ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), heartbeat_interval, + wal_sync_policy, ) .unwrap() } @@ -2790,6 +2829,51 @@ mod tests { stop(restarted, restarted_events, restarted_task).await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn buffered_wal_reports_append_without_durability_and_reopens_after_clean_shutdown() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start_autonomous_with_policy( + 0, + Duration::from_secs(60 * 60), + ShadowWalSyncPolicyV1::OnShutdown, + ); + wait_ready(&mut events).await; + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + + loop { + if let ShadowServiceEventV1::WalAppended { + batches, + records, + durable, + } = next_event(&mut events).await + { + assert_eq!(batches, 1); + assert!(records > 0); + assert!(!durable); + break; + } + } + stop(handle, events, task).await; + + let (restarted, mut restarted_events, restarted_task) = harness.start_autonomous(0); + let mut replayed = 0; + loop { + match next_event(&mut restarted_events).await { + ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches, + ShadowServiceEventV1::Ready { autonomous_clock } => { + assert!(autonomous_clock); + break; + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("buffered WAL restart failed: {error}") + } + _ => {} + } + } + assert_eq!(replayed, 1); + stop(restarted, restarted_events, restarted_task).await; + } + fn phase_carrier( author: AuthorityIndex, statement: RbcPhaseStatementV1, diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 38744dd2..0b8d6a50 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -79,6 +79,15 @@ impl Validator { "Starfish-RBC-DAG shadow mode requires consensus 'starfish-rbc'" )); } + if public_config + .parameters + .starfish_rbc_dag_shadow_buffered_wal + && !public_config.parameters.starfish_rbc_dag_shadow + { + return Err(eyre!( + "Starfish-RBC-DAG buffered benchmark WAL requires the RBC-DAG shadow" + )); + } if is_starfish_rbc { let protocol_instance = public_config .parameters @@ -222,12 +231,22 @@ impl Validator { } else { None }; - let starfish_rbc_dag_shadow_wal = - if public_config.parameters.starfish_rbc_dag_autonomous_clock { - private_config.starfish_rbc_dag_autonomous_clock_wal() - } else { - private_config.starfish_rbc_dag_shadow_wal() - }; + let starfish_rbc_dag_shadow_wal = if public_config + .parameters + .starfish_rbc_dag_shadow_buffered_wal + && public_config.parameters.starfish_rbc_dag_autonomous_clock + { + private_config.starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal() + } else if public_config + .parameters + .starfish_rbc_dag_shadow_buffered_wal + { + private_config.starfish_rbc_dag_shadow_buffered_benchmark_wal() + } else if public_config.parameters.starfish_rbc_dag_autonomous_clock { + private_config.starfish_rbc_dag_autonomous_clock_wal() + } else { + private_config.starfish_rbc_dag_shadow_wal() + }; let (core, bls_cert_aggregator) = Core::open( block_handler, @@ -412,6 +431,36 @@ mod smoke_tests { })); } + #[tokio::test] + async fn buffered_shadow_wal_requires_shadow_mode() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config + .parameters + .starfish_rbc_dag_shadow_buffered_wal = true; + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("buffered benchmark WAL requires the RBC-DAG shadow") + })); + } + #[tokio::test] async fn autonomous_clock_rejects_non_rbc_protocol() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 4f560e27..b22cbc30 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -193,6 +193,10 @@ enum Operation { /// Maximum interval between autonomous RBC-DAG heartbeat carriers. #[clap(long, value_name = "INT")] starfish_rbc_dag_heartbeat_interval_ms: Option, + /// Benchmark-only: write ordered shadow-WAL frames but force them to + /// stable storage only at clean shutdown. This run is not crash-safe. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_shadow_buffered_wal: bool, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, /// Dissemination mode override: @@ -306,6 +310,7 @@ async fn main() -> Result<()> { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_shadow_buffered_wal, duration_secs, dissemination_mode, } => { @@ -318,6 +323,8 @@ async fn main() -> Result<()> { node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; + node_parameters.starfish_rbc_dag_shadow_buffered_wal = + starfish_rbc_dag_shadow_buffered_wal; if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; } @@ -411,6 +418,10 @@ async fn local_benchmark( consensus_protocol: String, duration_secs: u64, ) -> Result<()> { + eyre::ensure!( + duration_secs > 0, + "benchmark duration must be greater than zero" + ); println!("\n=== Benchmark Configuration ==="); println!("Committee Size: {committee_size}"); println!("Byzantine Nodes: {num_byzantine_nodes}"); @@ -430,6 +441,16 @@ async fn local_benchmark( .unwrap_or("ed25519") } ); + if node_parameters.starfish_rbc_dag_shadow { + println!( + "Shadow WAL: {}", + if node_parameters.starfish_rbc_dag_shadow_buffered_wal { + "buffered benchmark profile (sync on clean shutdown; not crash-safe)" + } else { + "sync every transition (crash-safe reference profile)" + } + ); + } if let Some(latency) = node_parameters.uniform_latency_ms { println!("Network Latency: {latency} ms (uniform)"); } else { @@ -455,15 +476,15 @@ async fn local_benchmark( let ips = vec![IpAddr::V4(Ipv4Addr::LOCALHOST); committee_size]; let committee = Committee::new_for_benchmarks(committee_size); load /= committee.len(); - let parameters = Parameters::almost_default(load); + let mut parameters = Parameters::almost_default(load); + parameters.benchmark_duration = Some(Duration::from_secs(duration_secs)); // Equivocating Byzantine strategies must not generate transactions. - let byzantine_parameters = if ByzantineStrategy::from_strategy_str(&byzantine_strategy) + let mut byzantine_parameters = parameters.clone(); + if ByzantineStrategy::from_strategy_str(&byzantine_strategy) .is_some_and(|s| s.is_equivocating()) { - Parameters::almost_default(0) - } else { - parameters.clone() - }; + byzantine_parameters.load = 0; + } let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; let starfish_rbc_dag_autonomous_clock_expected = @@ -594,12 +615,36 @@ async fn local_benchmark( } } + // `duration_secs` is an active transaction-submission window, not a + // process-lifetime cutoff. Every finite generator holds metrics inactive + // through connection warmup, then opens this latch immediately before its + // first batch. Start the benchmark only after every honest validator has + // crossed that boundary. + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if metrics_of_honest_validators + .iter() + .all(|metrics| metrics.metrics_active.load(Ordering::Relaxed)) + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .wrap_err("transaction generators did not open the active benchmark window")?; + println!("Active transaction window started ({duration_secs} seconds)"); + let autonomous_clock_baselines = starfish_rbc_dag_autonomous_clock_expected.then(|| { metrics_of_honest_validators .iter() .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) .collect::>() }); + let counter_baselines = metrics_of_honest_validators + .iter() + .map(|metrics| metrics.local_benchmark_counter_baseline()) + .collect::>(); // Run for specified duration tokio::select! { @@ -617,6 +662,7 @@ async fn local_benchmark( starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, autonomous_clock_baselines.clone(), + Some(counter_baselines.clone()), ); // Abort all tasks @@ -644,6 +690,7 @@ async fn local_benchmark( starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, autonomous_clock_baselines, + Some(counter_baselines), ); fs::remove_dir_all(base_dir)?; eyre::bail!("All validators completed before the requested benchmark duration") @@ -956,6 +1003,7 @@ mod tests { "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-heartbeat-interval-ms", "125", + "--starfish-rbc-dag-shadow-buffered-wal", ]) .unwrap(); @@ -965,6 +1013,7 @@ mod tests { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_shadow_buffered_wal, .. } = args.operation else { @@ -975,6 +1024,7 @@ mod tests { assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); assert_eq!(starfish_rbc_dag_heartbeat_interval_ms, Some(125)); + assert!(starfish_rbc_dag_shadow_buffered_wal); } #[test] diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 8967f99a..4edae30e 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -7,7 +7,9 @@ The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That s implemented. The milestone-three direct-header comparison runtime is enabled with `--consensus starfish-rbc --starfish-rbc-dag-shadow`. Milestone four adds a separate control-only runtime with `--starfish-rbc-dag-autonomous-clock`; its carrier rounds advance independently through -authenticated admission and empty heartbeats. Both modes leave the direct prototype's DAG, +authenticated admission and empty heartbeats. Performance experiments may add +`--starfish-rbc-dag-shadow-buffered-wal` to remove per-transition disk synchronization; that +profile is explicitly not crash-safe. Both modes leave the direct prototype's DAG, pacemaker, commit, and output unchanged. The eventual protocol is new, not a transport option or a version-two alias for `starfish-rbc`. @@ -81,13 +83,15 @@ handle even after its async supervisor is detached. A same-process restart again therefore forbidden until the worker has exited; process exit remains safe. A production-quality same-process restart path needs an operating-system file lock or a fully cancellable storage task. -The shadow runtime is not a proof or a performance implementation. It intentionally fsyncs every -accepted transition and its reference reducer clones retained model/journal history, so total CPU -work grows superlinearly with a long run. It also uses a fixed unsolicited-retention window only as -a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the -composition, resource bounds, and performance path are completed, `starfish-rbc-dag` must be -described as an experimental shadow/reference implementation rather than a proven signature-free -Starfish variant or a fair throughput baseline. +The shadow runtime is not a proof or a production performance implementation. Its default +crash-safe profile intentionally fsyncs every accepted transition, and its reference reducer clones +retained model/journal history. A separate explicit benchmark profile writes the same ordered, +checksummed frames but syncs them only on clean shutdown; it reports appended and durable records +separately and makes no crash-safety claim. This removes the known persistence observer effect +without changing the protocol reducer. The runtime also uses a fixed unsolicited-retention window +only as a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the +composition and resource bounds are completed, `starfish-rbc-dag` remains an experimental +shadow/reference implementation rather than a proven signature-free Starfish variant. The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not @@ -778,9 +782,10 @@ newest current-process observation (`<= 4`). These are empirical benchmark cover asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as proof of a protocol failure. -The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a -synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars and -allocator overhead). Mirror mode budgets one peer fan-in plus five local/control inputs and accepts +The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a slow +reference transition (including synchronous fsync in the crash-safe profile), capping queued +maximum-sized carrier bodies at 256 MiB (plus sidecars and allocator overhead). Mirror mode budgets +one peer fan-in plus five local/control inputs and accepts at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, @@ -789,7 +794,7 @@ per peer. Requested historical slots remain recoverable beyond the benchmark-onl retention window. Autonomous benchmark validity is separate from delivery comparison validity. -`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the WAL and heartbeat counters must progress, +`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and heartbeat counters must progress, the carrier round and embedded-RBC delivery count must advance during the measured interval, recovery must drain, and the reported clock-state/backlog and cross-node skew must remain within the configured empirical guards. These checks establish that the observational carrier plane stayed @@ -895,20 +900,32 @@ state. Batching can reduce the number of separately scheduled RBC control messages, but it does not remove their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout -transport is added. Shadow mode also sends both direct and embedded transcripts and is a correctness -instrument, not a performance result. In milestone three it additionally fsyncs each accepted -transition and validates through a clone-based reference reducer. Those costs are deliberately not -charged as protocol overhead: performance runs require incremental state transitions/checkpoints -or an equivalently durable baseline, plus separate WAL/fsync accounting. - -As an implementation-continuity check, a 10-validator, 60-second local run on 2026-08-11 used the -AWS RTT emulator, a nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. -All validators reached carrier round 196 with zero skew and zero pending recovery; the control -plane recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative direct -Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 end-to-end -latency, and 0.45 MB/s average outbound bandwidth. This is not a comparative performance claim: -the cutoff includes the local generator warmup, and the control-only shadow still performs -per-transition fsync/reference-model work on the authoritative network socket. +transport is added. Shadow mode also sends both direct and embedded transcripts. The default +crash-safe reference profile fsyncs each accepted transition and validates through a clone-based +reducer; it is a correctness/replay instrument, not a protocol-performance result. The explicit +buffered-WAL profile keeps the exact framed event path but syncs only on clean shutdown and therefore +cannot be used for crash-safety claims. Benchmark output reports appended and durable WAL work +separately. The clone-based reducer remains intentionally unoptimized until measurement shows it +matters. + +A matched 10-validator local A/B on 2026-08-11 used a full 60-second active transaction window, +the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous +heartbeat. The harness waits through generator warmup, snapshots cumulative counters at the active +boundary, and drains final latency samples. + +| Profile | Verdict | TPS | p50 block | p50 E2E | Outbound | +|---|---:|---:|---:|---:|---:| +| Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | +| Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | +| Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | + +The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted +heartbeats, 27,180 embedded-RBC deliveries, 27,424 appended batches, zero round skew, and zero +pending recovery. Its latency and throughput match the shadow-off baseline while making the +expected extra carrier traffic visible. The crash-safe run shed shadow work and is reported only +as a diagnostic: it isolates synchronous persistence as a severe observer effect and must not be +cited as a protocol result. Neither run measures application latency through the carrier DAG yet, +because the direct Starfish-RBC path remains authoritative in milestone four. ## 19. Contained implementation milestones From 57d660a7506a3c37291663db789f75cb1d333bbb Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:23:19 +0200 Subject: [PATCH 33/62] Make carrier DAG RBC authoritative --- README.md | 72 ++- crates/orchestrator/src/main.rs | 20 +- crates/orchestrator/src/protocol/starfish.rs | 13 +- crates/starfish-core/src/config.rs | 32 +- crates/starfish-core/src/metrics.rs | 99 +++- crates/starfish-core/src/net_sync.rs | 68 ++- crates/starfish-core/src/starfish_rbc.rs | 4 +- .../src/starfish_rbc_dag/journal.rs | 1 + .../starfish-core/src/starfish_rbc_dag/mod.rs | 213 ++++++++- .../src/starfish_rbc_dag/model.rs | 1 + .../src/starfish_rbc_dag/projection.rs | 1 + .../src/starfish_rbc_dag_shadow.rs | 100 +++- .../src/starfish_rbc_dag_shadow_service.rs | 439 ++++++++++++++++-- .../starfish-core/src/starfish_rbc_service.rs | 106 ++++- crates/starfish-core/src/validator.rs | 110 +++-- crates/starfish/src/main.rs | 68 +-- docs/starfish-rbc-dag-protocol.md | 85 ++-- 17 files changed, 1152 insertions(+), 280 deletions(-) diff --git a/README.md b/README.md index a0fdb39f..30f59c49 100644 --- a/README.md +++ b/README.md @@ -50,25 +50,23 @@ Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctnes prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). **Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate logical projection. Its -canonical types, deterministic models, crash journal, direct-header comparison shadow, and a -separate opt-in autonomous heartbeat carrier clock are implemented. Run the comparison shadow with -`--consensus starfish-rbc --starfish-rbc-dag-shadow`. Add -`--starfish-rbc-dag-autonomous-clock` to run an independent control-only carrier clock (prototype -heartbeat default: 250 ms); that mode deliberately does not map direct application headers to -carrier rounds or claim a direct-delivery comparison. Direct Starfish-RBC remains solely -authoritative in both modes, and shadow failures or results cannot affect proposals, commits, or -output as protocol state. Shadow traffic still shares the validator's network socket and bandwidth, -so it can perturb timing, and it must be enabled only on a homogeneous new-binary committee; there -is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` selector is not -implemented yet. The default shadow uses per-transition fsync and a clone-based reference reducer, -so it is a correctness instrument, not a fair performance baseline, and carries no safety or -liveness claim. Benchmark runs may add `--starfish-rbc-dag-shadow-buffered-wal`; this preserves the -ordered checksummed log and syncs it on clean shutdown, but deliberately gives up crash safety for -that run. Appended and durably synchronized WAL records are reported separately. -Its WAL can reopen the shadow actor, but this is not full validator crash recovery: authoritative -direct Starfish-RBC phase and delivery locks are not durable yet, so that baseline remains fail-stop -across process restart. The design and proof obligations are documented in the -[protocol design](docs/starfish-rbc-dag-protocol.md). +canonical types, deterministic models, crash journal, comparison shadow, autonomous carrier clock, +and opt-in authoritative embedded-RBC path are implemented. Run the comparison shadow with +`--consensus starfish-rbc --starfish-rbc-dag-shadow`; add +`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact +application headers in version-two carriers and make embedded ECHO/READY/delivery their sole +certification authority. Direct INIT remains payload transport, but direct ECHO/READY cannot clean +blocks in that mode. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for +Starfish-RBC by default); application and encodable phase carriers are emitted immediately. + +The current milestone changes certification, not consensus: the existing Starfish DAG still +consumes the embedded deliveries and retains its clean-predecessor proposal gate. The certified +carrier projection and frontier linearizer are the next milestones. Shadow traffic shares the +validator's network socket and bandwidth, and deployment requires a homogeneous new-binary +committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; +`--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean +shutdown and therefore forfeits crash safety. Full validator crash recovery also remains out of +scope. See the [protocol design](docs/starfish-rbc-dag-protocol.md). For a direct-header shadow comparison, `starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded observational path was disabled or shed work and the comparison must be discarded. Healthy live @@ -79,27 +77,25 @@ the cumulative direct and shadow delivery counters. Autonomous runs instead requ in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 in autonomous mode. -A matched 10-validator, 60-second-active-window local A/B on 2026-08-11 used the AWS RTT emulator, -nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. +A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, +nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared +600 ms leader/idle-carrier timeout. | Profile | Verdict | TPS | Block latency | E2E latency | Outbound BW | |---|---:|---:|---:|---:|---:| | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | -| Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | - -Every shadow validator reached carrier round 275 with zero skew and pending recovery; the run -recorded 2,749 heartbeats, 27,180 embedded-RBC deliveries, and 27,424 WAL batches. Thus this -prototype's carrier/RBC work had no measurable latency penalty in this run; the extra shadow -traffic cost about 0.05 MB/s outbound. The result is not yet application latency through RBC-DAG -because direct Starfish-RBC remains authoritative. - -The same 250 ms experiment with the default crash-safe, per-transition-fsync WAL shed shadow work -and ended `INVALID` (9/10 valid validators); its authoritative path slowed to 2,569.1 ms block and -3,067.4 ms end-to-end latency. This isolates synchronous shadow persistence as the prior observer -effect, rather than a carrier-DAG latency regression. Use the buffered profile for protocol -benchmarks and the default profile for crash/replay tests; do not cite an `INVALID` run as a -protocol result. The local harness now starts its timer after transaction-generator warmup, -subtracts warmup counters, and drains the final latency samples. +| Autonomous comparison, direct RBC authoritative | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | +| Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | + +The milestone-five run produced 10,722 embedded application deliveries, reached carrier rounds +458–459, and ended with zero pending recovery. It also proves that the earlier 250 ms experimental +heartbeat was not the latency cause: application and phase carriers are already event-driven, and +using the shared 600 ms timeout did not restore the direct baseline. The remaining slowdown is an +expected warning about the transitional architecture—the old direct DAG still serializes proposal +creation on embedded RBC cleanliness. Do not tune that obsolete gate; milestone six must let the +optimistic carrier clock advance independently and feed only certified vertices into consensus. +The local harness starts its timer after transaction-generator warmup, subtracts warmup counters, +and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's @@ -307,14 +303,14 @@ cargo run --release --bin starfish -- local-benchmark \ Additional flags: `--dissemination-mode`, `--adversarial-latency`, `--uniform-latency-ms`. -For the non-authoritative autonomous RBC-DAG benchmark profile: +For the authoritative embedded-RBC benchmark profile: ```bash cargo run --release --bin starfish -- local-benchmark \ --committee-size 10 --load 1000 --consensus starfish-rbc \ --block-authentication mac --mimic-extra-latency \ --starfish-rbc-dag-shadow --starfish-rbc-dag-autonomous-clock \ - --starfish-rbc-dag-heartbeat-interval-ms 250 \ + --starfish-rbc-dag-embedded-rbc-authority \ --starfish-rbc-dag-shadow-buffered-wal --duration-secs 60 ``` diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index b00903d9..761eb399 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -73,9 +73,10 @@ pub struct Opts { #[clap(long, global = true)] starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT", global = true)] - starfish_rbc_dag_heartbeat_interval_ms: Option, + /// Let embedded carrier ECHO/READY delivery certify Starfish-RBC + /// application headers. Requires autonomous RBC-DAG mode. + #[clap(long, global = true)] + starfish_rbc_dag_embedded_rbc_authority: bool, /// Benchmark-only: sync the framed shadow WAL only at clean shutdown. /// This removes per-transition disk pressure and is not crash-safe. @@ -91,7 +92,7 @@ pub struct Opts { struct StarfishRbcDagOverrides { shadow: bool, autonomous_clock: bool, - heartbeat_interval_ms: Option, + embedded_rbc_authority: bool, buffered_wal: bool, } @@ -896,8 +897,8 @@ fn load_benchmark_configs( if starfish_rbc_dag.autonomous_clock { node_parameters.starfish_rbc_dag_autonomous_clock = true; } - if let Some(interval_ms) = starfish_rbc_dag.heartbeat_interval_ms { - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + if starfish_rbc_dag.embedded_rbc_authority { + node_parameters.starfish_rbc_dag_embedded_rbc_authority = true; } if starfish_rbc_dag.buffered_wal { node_parameters.starfish_rbc_dag_shadow_buffered_wal = true; @@ -1085,7 +1086,7 @@ async fn run( let starfish_rbc_dag = StarfishRbcDagOverrides { shadow: opts.starfish_rbc_dag_shadow, autonomous_clock: opts.starfish_rbc_dag_autonomous_clock, - heartbeat_interval_ms: opts.starfish_rbc_dag_heartbeat_interval_ms, + embedded_rbc_authority: opts.starfish_rbc_dag_embedded_rbc_authority, buffered_wal: opts.starfish_rbc_dag_shadow_buffered_wal, }; match opts.operation { @@ -2383,8 +2384,7 @@ mod tests { "mac", "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", - "--starfish-rbc-dag-heartbeat-interval-ms", - "125", + "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-dag-shadow-buffered-wal", "--protocols", "starfish-rbc", @@ -2394,8 +2394,8 @@ mod tests { assert_eq!(opts.block_authentication.as_deref(), Some("mac")); assert!(opts.starfish_rbc_dag_shadow); assert!(opts.starfish_rbc_dag_autonomous_clock); + assert!(opts.starfish_rbc_dag_embedded_rbc_authority); assert!(opts.starfish_rbc_dag_shadow_buffered_wal); - assert_eq!(opts.starfish_rbc_dag_heartbeat_interval_ms, Some(125)); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); }; diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 1bb38995..447b6e56 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -324,9 +324,8 @@ impl StarfishProtocol { // validator configuration. node_parameters.starfish_rbc_dag_shadow = false; node_parameters.starfish_rbc_dag_autonomous_clock = false; + node_parameters.starfish_rbc_dag_embedded_rbc_authority = false; node_parameters.starfish_rbc_dag_shadow_buffered_wal = false; - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = - config::node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(); } node_parameters } @@ -363,14 +362,13 @@ impl StarfishProtocol { mod tests { use super::{ProtocolMetrics, StarfishNodeParameters, StarfishProtocol}; use crate::{benchmark::BenchmarkParameters, client::Instance}; - use starfish_core::config::{NodeParameters, node_defaults}; + use starfish_core::config::NodeParameters; #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); @@ -384,7 +382,6 @@ mod tests { assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); assert!(parameters.starfish_rbc_dag_shadow_buffered_wal); - assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } #[test] @@ -392,7 +389,6 @@ mod tests { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); @@ -411,11 +407,6 @@ mod tests { !parameters.starfish_rbc_dag_shadow_buffered_wal, "a global buffered-WAL flag must not leak into non-RBC comparison members" ); - assert_eq!( - parameters.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), - "a global heartbeat override must not leak into non-RBC comparison members" - ); } #[test] diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 66f0a305..8b7de0b0 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -77,10 +77,11 @@ pub struct NodeParameters { /// `starfish_rbc_dag_shadow`. #[serde(default)] pub starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous Starfish-RBC-DAG heartbeat - /// carriers. The value is ignored unless the autonomous clock is enabled. - #[serde(default = "node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms")] - pub starfish_rbc_dag_heartbeat_interval_ms: u64, + /// Use embedded carrier ECHO/READY delivery as the certification authority + /// for Starfish-RBC application headers. Direct RBC retains INIT/payload + /// transport but its phase messages cannot mark a block clean. + #[serde(default)] + pub starfish_rbc_dag_embedded_rbc_authority: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but /// calls `sync_all` only during clean shutdown. This removes persistence /// pressure from latency experiments and deliberately forfeits the @@ -137,10 +138,6 @@ pub mod node_defaults { 5 } - pub fn default_starfish_rbc_dag_heartbeat_interval_ms() -> u64 { - 250 - } - pub fn default_causal_push_shard_round_lag() -> RoundNumber { 0 } @@ -168,8 +165,7 @@ impl Default for NodeParameters { starfish_rbc_protocol_instance: None, starfish_rbc_dag_shadow: false, starfish_rbc_dag_autonomous_clock: false, - starfish_rbc_dag_heartbeat_interval_ms: - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), + starfish_rbc_dag_embedded_rbc_authority: false, starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: @@ -429,9 +425,9 @@ impl ImportExport for NodePrivateConfig {} #[cfg(test)] mod tests { - use std::path::Path; + use std::{path::Path, time::Duration}; - use super::{NodeParameters, NodePrivateConfig, node_defaults}; + use super::{NodeParameters, NodePrivateConfig}; #[test] fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { @@ -440,10 +436,6 @@ mod tests { assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); - assert_eq!( - parameters.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() - ); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); assert_ne!(protocol_instance, [0; 32]); @@ -457,10 +449,6 @@ mod tests { assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); - assert_eq!( - decoded.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() - ); } #[test] @@ -468,7 +456,7 @@ mod tests { let parameters = NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, + leader_timeout: Duration::from_millis(125), starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -478,7 +466,7 @@ mod tests { assert!(decoded.starfish_rbc_dag_shadow); assert!(decoded.starfish_rbc_dag_autonomous_clock); assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); - assert_eq!(decoded.starfish_rbc_dag_heartbeat_interval_ms, 125); + assert_eq!(decoded.leader_timeout, Duration::from_millis(125)); } #[test] diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index a00075c0..5d47fc66 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -267,6 +267,7 @@ pub struct MetricReporter { pub struct AutonomousClockBenchmarkBaseline { accepted_heartbeats: u64, delivered_carriers: u64, + delivered_applications: u64, wal_batches: u64, wal_records: u64, carrier_round: i64, @@ -291,6 +292,7 @@ struct AutonomousClockBenchmarkSummary { bounded_nodes: usize, accepted_heartbeats: u64, delivered_carriers: u64, + delivered_applications: u64, wal_batches: u64, wal_records: u64, pending_recovery: i64, @@ -309,6 +311,7 @@ fn summarize_autonomous_clock_benchmark( metrics: &[Arc], committee_size: usize, baselines: Option<&[AutonomousClockBenchmarkBaseline]>, + embedded_rbc_authority: bool, ) -> AutonomousClockBenchmarkSummary { let committee_size = i64::try_from(committee_size).unwrap_or(i64::MAX); let maximum_phase_backlog_bound = @@ -338,6 +341,12 @@ fn summarize_autonomous_clock_benchmark( .with_label_values(&["delivery", "shadow"]) .get() > baseline.delivered_carriers + && (!embedded_rbc_authority + || metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .get() + > baseline.delivered_applications) && metrics .starfish_rbc_dag_shadow_wal_appended_batches_total .get() @@ -385,6 +394,15 @@ fn summarize_autonomous_clock_benchmark( .get() }) .sum(); + let delivered_applications = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .get() + }) + .sum(); let wal_batches = metrics .iter() .map(|metrics| { @@ -451,6 +469,7 @@ fn summarize_autonomous_clock_benchmark( bounded_nodes, accepted_heartbeats, delivered_carriers, + delivered_applications, wal_batches, wal_records, pending_recovery, @@ -487,6 +506,10 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .get(), + delivered_applications: self + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .get(), wal_batches: self .starfish_rbc_dag_shadow_wal_appended_batches_total .get(), @@ -1406,6 +1429,7 @@ impl Metrics { committee_size: usize, starfish_rbc_dag_shadow_expected: bool, starfish_rbc_dag_autonomous_clock_expected: bool, + starfish_rbc_dag_embedded_rbc_authority_expected: bool, autonomous_clock_baselines: Option>, counter_baselines: Option>, ) { @@ -1653,11 +1677,18 @@ impl Metrics { &metrics, committee_size, autonomous_clock_baselines.as_deref(), + starfish_rbc_dag_embedded_rbc_authority_expected, ); let round_lag = summary.maximum_round.saturating_sub(summary.minimum_round); table.add_row(row![bH2->""]); - table.add_row(row![bH2->"RBC-DAG Autonomous Clock Verification"]); + table.add_row(row![ + bH2->if starfish_rbc_dag_embedded_rbc_authority_expected { + "RBC-DAG Embedded RBC Authority Verification" + } else { + "RBC-DAG Autonomous Clock Verification" + } + ]); table.add_row(row![ b->"Clock verdict:", if summary.verdict_valid { @@ -1681,9 +1712,10 @@ impl Metrics { table.add_row(row![ b->"Clock/WAL progress:", format!( - "heartbeats={}, RBC deliveries={}, WAL batches={}, records={}, open rounds={}..{}", + "heartbeats={}, carrier deliveries={}, application deliveries={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, summary.delivered_carriers, + summary.delivered_applications, summary.wal_batches, summary.wal_records, summary.minimum_round, @@ -2279,7 +2311,7 @@ mod tests { autonomous_clock_metrics(11, 6, 1), ]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); assert!(summary.verdict_valid); assert_eq!(summary.valid_nodes, 4); @@ -2304,7 +2336,10 @@ mod tests { .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) .collect::>(); - assert!(!summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); + assert!( + !summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) + .verdict_valid + ); for metrics in &metrics { metrics @@ -2330,7 +2365,59 @@ mod tests { metrics.starfish_rbc_dag_shadow_carrier_round.inc(); } - assert!(summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); + assert!( + summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) + .verdict_valid + ); + } + + #[test] + fn embedded_authority_summary_requires_application_delivery_progress() { + let metrics = vec![autonomous_clock_metrics(8, 0, 0)]; + let baselines = metrics + .iter() + .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) + .collect::>(); + let metrics = &metrics[0]; + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_batches_total + .inc(); + metrics + .starfish_rbc_dag_shadow_wal_appended_records_total + .inc(); + metrics.starfish_rbc_dag_shadow_carrier_round.inc(); + + assert!( + !summarize_autonomous_clock_benchmark( + &[Arc::clone(metrics)], + 2, + Some(&baselines), + true, + ) + .verdict_valid + ); + + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .inc(); + assert!( + summarize_autonomous_clock_benchmark( + &[Arc::clone(metrics)], + 2, + Some(&baselines), + true, + ) + .verdict_valid + ); } #[test] @@ -2349,7 +2436,7 @@ mod tests { ); let metrics = vec![no_progress, invalid_clock, unbounded]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); assert!(!summary.verdict_valid); assert_eq!(summary.valid_nodes, 2); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 2479ab54..74dc8e4b 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -45,7 +45,7 @@ use crate::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, - starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, + starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader, RbcCommitteeId, RbcProtocolInstanceId}, starfish_rbc_dag::{ RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, storage::ShadowWalSyncPolicyV1, @@ -59,7 +59,8 @@ use crate::{ start_starfish_rbc_dag_shadow_service_v1, }, starfish_rbc_service::{ - RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, + RbcInitialAuthenticator, RbcPhaseAuthorityV1, RbcServiceEvent, RbcServiceHandle, + start_starfish_rbc_service_with_phase_authority, }, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ @@ -1809,13 +1810,7 @@ impl NetworkSyncer let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); - let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow - && node_parameters.starfish_rbc_dag_autonomous_clock - { - // Autonomous carrier rounds are independent of direct consensus - // rounds and recover entirely from their distinct WAL. - Some(Vec::new()) - } else if node_parameters.starfish_rbc_dag_shadow { + let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { match recovered_local_rbc_headers(&core) { Ok(headers) => Some(headers), Err(error) => { @@ -1880,7 +1875,12 @@ impl NetworkSyncer } BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, }; - let (service, events, task) = start_starfish_rbc_service( + let phase_authority = if node_parameters.starfish_rbc_dag_embedded_rbc_authority { + RbcPhaseAuthorityV1::EmbeddedCarrierDag + } else { + RbcPhaseAuthorityV1::Direct + }; + let (service, events, task) = start_starfish_rbc_service_with_phase_authority( committee.clone(), dag_state.get_own_authority_index(), protocol_instance, @@ -1889,6 +1889,7 @@ impl NetworkSyncer initial_authenticator, dag_state.highest_round(), STARFISH_RBC_HEADER_RETRY_INTERVAL, + phase_authority, ) .expect("validated Starfish-RBC configuration must start its service"); (Some(service), Some(events), Some(task)) @@ -1947,9 +1948,11 @@ impl NetworkSyncer dag_state.get_own_authority_index(), context, authorizer, - Duration::from_millis( - node_parameters.starfish_rbc_dag_heartbeat_interval_ms, - ), + recovered_local_headers, + // The idle carrier pacemaker deliberately shares the + // resolved Starfish leader timeout. Application and + // embedded RBC phase carriers remain event-driven. + node_parameters.leader_timeout, wal_sync_policy, ) } else { @@ -2199,6 +2202,11 @@ impl NetworkSyncer }) }); + let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; + let embedded_rbc_committee_id = embedded_rbc_authority.then(|| { + RbcCommitteeId::derive(&inner.committee) + .expect("validated direct RBC committee must retain a stable identifier") + }); let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); let shadow_metrics = metrics.clone(); @@ -2254,6 +2262,40 @@ impl NetworkSyncer .inc(); tracing::debug!(?identity, "RBC-DAG shadow delivered carrier"); } + ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header } => { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .inc(); + tracing::debug!( + ?carrier, + application = ?header.reference(), + "RBC-DAG shadow delivered embedded application header" + ); + if embedded_rbc_authority { + match PinnedRbcHeader::validate_with_committee_id( + header, + &event_inner.committee, + embedded_rbc_committee_id + .expect("embedded authority must cache its committee ID"), + ) { + Ok(header) => { + event_inner + .syncer + .apply_starfish_rbc_deliveries(vec![header]) + .await; + } + Err(error) => { + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + tracing::error!( + ?carrier, + ?error, + "Embedded RBC delivered an invalid application header" + ); + } + } + } + } ShadowServiceEventV1::ComparisonBacklog { unpaired_direct, unpaired_shadow, diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index 684f0995..b6ace62c 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -445,7 +445,7 @@ pub(crate) struct PinnedRbcHeader { } impl PinnedRbcHeader { - fn validate_with_committee_id( + pub(crate) fn validate_with_committee_id( header: RbcCanonicalHeader, committee: &Committee, committee_id: RbcCommitteeId, @@ -651,7 +651,7 @@ impl fmt::Debug for RbcProtocolInstanceId { pub(crate) struct RbcCommitteeId([u8; COMMITTEE_ID_SIZE]); impl RbcCommitteeId { - fn derive(committee: &Committee) -> Result { + pub(crate) fn derive(committee: &Committee) -> Result { if committee.len() > MAX_COMMITTEE_SIZE as usize { return Err(RbcError::CommitteeTooLarge(committee.len())); } diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index 24812f40..bdb4fb1d 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -1229,6 +1229,7 @@ mod tests { own_prev: parent(author), weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + application_header: None, data_acknowledgments: Vec::new(), phase_batch, consensus_vertex, diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs index 31f82af0..f734e527 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -27,6 +27,7 @@ use crate::{ MacTag, MlDsa44SignatureBytes, MlDsa44Signer, MlDsa65SignatureBytes, MlDsa65Signer, SIGNATURE_SIZE, SignatureBytes, Signer, TransactionsCommitment, }, + starfish_rbc::RbcCanonicalHeader, types::{ AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, TimestampNs, @@ -35,6 +36,10 @@ use crate::{ pub const CARRIER_FORMAT_VERSION_V1: u8 = 1; pub const CARRIER_WIRE_FORMAT_VERSION_V1: u8 = 0x81; +/// V2 extends V1 with one complete canonical application header. Control-only +/// carriers remain byte-for-byte V1 so existing autonomous-clock WALs reopen. +pub const CARRIER_FORMAT_VERSION_V2: u8 = 2; +pub const CARRIER_WIRE_FORMAT_VERSION_V2: u8 = 0x82; pub const MAX_CARRIER_CONTENT_SIZE_V1: usize = 4 * 1024 * 1024; pub const MAX_PHASE_STATEMENTS_V1: usize = 2_048; @@ -48,6 +53,7 @@ const ACKNOWLEDGMENTS_FIELD: u8 = 0x06; const PHASE_BATCH_FIELD: u8 = 0x07; const CONSENSUS_VERTEX_FIELD: u8 = 0x08; const CREATION_TIME_FIELD: u8 = 0x09; +const APPLICATION_HEADER_FIELD: u8 = 0x0A; const CONSENSUS_ROUND_FIELD: u8 = 0x01; const STRONG_PARENTS_FIELD: u8 = 0x02; const DELIVERY_FRONTIER_FIELD: u8 = 0x03; @@ -317,6 +323,7 @@ pub struct CarrierHeaderV1 { own_prev: BlockReference, weak_parents: Vec, transactions_commitment: TransactionsCommitment, + application_header: Option, data_acknowledgments: Vec, phase_batch: Vec, consensus_vertex: Option, @@ -330,6 +337,9 @@ pub struct CarrierHeaderV1Args { pub own_prev: BlockReference, pub weak_parents: Vec, pub transactions_commitment: TransactionsCommitment, + /// Exact application header disseminated by this carrier. `None` denotes + /// a control-only heartbeat and retains the frozen V1 byte grammar. + pub application_header: Option, pub data_acknowledgments: Vec, pub phase_batch: Vec, pub consensus_vertex: Option, @@ -344,6 +354,7 @@ impl CarrierHeaderV1 { own_prev: args.own_prev, weak_parents: args.weak_parents, transactions_commitment: args.transactions_commitment, + application_header: args.application_header, data_acknowledgments: args.data_acknowledgments, phase_batch: args.phase_batch, consensus_vertex: args.consensus_vertex, @@ -373,6 +384,10 @@ impl CarrierHeaderV1 { self.transactions_commitment } + pub fn application_header(&self) -> Option<&RbcCanonicalHeader> { + self.application_header.as_ref() + } + pub fn data_acknowledgments(&self) -> &[BlockReference] { &self.data_acknowledgments } @@ -1542,6 +1557,12 @@ pub enum RbcDagError { InvalidWeakParent(BlockReference), WeakParentsNotOrdered, InvalidCarrierThreshold, + InvalidApplicationHeader, + ApplicationAuthorMismatch { + carrier: AuthorityIndex, + application: AuthorityIndex, + }, + ApplicationCommitmentMismatch, InvalidAcknowledgment(BlockReference), DuplicateAcknowledgment(BlockReference), InvalidPhaseTarget(BlockReference), @@ -1641,6 +1662,22 @@ fn validate_outer_header( return Err(RbcDagError::InvalidCarrierThreshold); } + if let Some(application_header) = &header.application_header { + application_header + .validate_for_committee(committee) + .map_err(|_| RbcDagError::InvalidApplicationHeader)?; + let application = application_header.reference().authority; + if application != header.author { + return Err(RbcDagError::ApplicationAuthorMismatch { + carrier: header.author, + application, + }); + } + if application_header.transactions_commitment() != header.transactions_commitment { + return Err(RbcDagError::ApplicationCommitmentMismatch); + } + } + if header.data_acknowledgments.len() > u16::MAX as usize { return Err(RbcDagError::VectorTooLong { field: "acknowledgments", @@ -1736,10 +1773,14 @@ fn encode_header( ) -> Result, RbcDagError> { let mut bytes = Vec::new(); bytes.push(CONTENT_FORMAT_FIELD); - bytes.push(match acknowledgment_encoding { - AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, - AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, - }); + bytes.push( + match (acknowledgment_encoding, header.application_header.is_some()) { + (AckEncoding::Expanded, false) => CARRIER_FORMAT_VERSION_V1, + (AckEncoding::Compressed, false) => CARRIER_WIRE_FORMAT_VERSION_V1, + (AckEncoding::Expanded, true) => CARRIER_FORMAT_VERSION_V2, + (AckEncoding::Compressed, true) => CARRIER_WIRE_FORMAT_VERSION_V2, + }, + ); bytes.push(AUTHOR_FIELD); bytes.extend_from_slice(&header.author.to_be_bytes()); bytes.push(CARRIER_ROUND_FIELD); @@ -1753,6 +1794,10 @@ fn encode_header( } bytes.push(TRANSACTIONS_COMMITMENT_FIELD); bytes.extend_from_slice(header.transactions_commitment.as_ref()); + if let Some(application_header) = &header.application_header { + bytes.push(APPLICATION_HEADER_FIELD); + encode_application_header(&mut bytes, application_header)?; + } bytes.push(ACKNOWLEDGMENTS_FIELD); match acknowledgment_encoding { AckEncoding::Expanded => { @@ -1796,6 +1841,37 @@ fn encode_header( Ok(bytes) } +fn encode_application_header( + bytes: &mut Vec, + header: &RbcCanonicalHeader, +) -> Result<(), RbcDagError> { + let reference = header.reference(); + bytes.push(0x01); + bytes.extend_from_slice(&reference.authority.to_be_bytes()); + bytes.push(0x02); + bytes.extend_from_slice(&reference.round.to_be_bytes()); + bytes.push(0x03); + encode_count( + bytes, + "application parents", + header.block_references().len(), + )?; + for parent in header.block_references() { + encode_reference(bytes, *parent); + } + let acknowledgments = header.acknowledgment_references(); + bytes.push(0x04); + encode_count(bytes, "application acknowledgments", acknowledgments.len())?; + for acknowledgment in acknowledgments { + encode_reference(bytes, acknowledgment); + } + bytes.push(0x05); + bytes.extend_from_slice(&header.meta_creation_time_ns().to_be_bytes()); + bytes.push(0x06); + bytes.extend_from_slice(header.transactions_commitment().as_ref()); + Ok(()) +} + fn encode_consensus_vertex( bytes: &mut Vec, vertex: &ConsensusVertexV1, @@ -1908,13 +1984,15 @@ fn decode_header( let mut decoder = Decoder::new(bytes); decoder.expect_marker(CONTENT_FORMAT_FIELD)?; let version = decoder.read_u8()?; - let expected_version = match acknowledgment_encoding { - AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, - AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, + let has_application_header = match (acknowledgment_encoding, version) { + (AckEncoding::Expanded, CARRIER_FORMAT_VERSION_V1) + | (AckEncoding::Compressed, CARRIER_WIRE_FORMAT_VERSION_V1) => false, + (AckEncoding::Expanded, CARRIER_FORMAT_VERSION_V2) + | (AckEncoding::Compressed, CARRIER_WIRE_FORMAT_VERSION_V2) => true, + _ => { + return Err(RbcDagError::UnsupportedVersion(version)); + } }; - if version != expected_version { - return Err(RbcDagError::UnsupportedVersion(version)); - } decoder.expect_marker(AUTHOR_FIELD)?; let author = decoder.read_u16()?; decoder.expect_marker(CARRIER_ROUND_FIELD)?; @@ -1926,6 +2004,12 @@ fn decode_header( let weak_parents = decoder.read_references(weak_count)?; decoder.expect_marker(TRANSACTIONS_COMMITMENT_FIELD)?; let transactions_commitment = TransactionsCommitment::from_bytes(decoder.read_array()?); + let application_header = if has_application_header { + decoder.expect_marker(APPLICATION_HEADER_FIELD)?; + Some(decoder.read_application_header()?) + } else { + None + }; decoder.expect_marker(ACKNOWLEDGMENTS_FIELD)?; let data_acknowledgments = match acknowledgment_encoding { AckEncoding::Expanded => { @@ -1950,6 +2034,7 @@ fn decode_header( own_prev, weak_parents: weak_parents.clone(), transactions_commitment, + application_header: application_header.clone(), data_acknowledgments: acknowledgments.clone(), phase_batch: Vec::new(), consensus_vertex: None, @@ -1990,6 +2075,7 @@ fn decode_header( own_prev, weak_parents, transactions_commitment, + application_header, data_acknowledgments, phase_batch, consensus_vertex, @@ -2042,6 +2128,33 @@ impl<'a> Decoder<'a> { Ok(u64::from_be_bytes(self.read_array()?)) } + fn read_application_header(&mut self) -> Result { + self.expect_marker(0x01)?; + let author = self.read_u16()?; + self.expect_marker(0x02)?; + let round = self.read_u32()?; + self.expect_marker(0x03)?; + let parent_count = self.read_count("application parents", u16::MAX as usize)?; + let parents = self.read_references(parent_count)?; + self.expect_marker(0x04)?; + let acknowledgment_count = + self.read_count("application acknowledgments", u16::MAX as usize)?; + let acknowledgments = self.read_references(acknowledgment_count)?; + self.expect_marker(0x05)?; + let creation_time_ns = self.read_u64()?; + self.expect_marker(0x06)?; + let transactions_commitment = TransactionsCommitment::from_bytes(self.read_array()?); + RbcCanonicalHeader::try_new( + author, + round, + parents, + acknowledgments, + creation_time_ns, + transactions_commitment, + ) + .map_err(|_| RbcDagError::InvalidApplicationHeader) + } + fn expect_marker(&mut self, expected: u8) -> Result<(), RbcDagError> { let actual = self.read_u8()?; if actual != expected { @@ -2183,6 +2296,7 @@ mod tests { use crate::crypto::{ dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }; + use crate::types::VerifiedBlock; fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { BlockReference { @@ -2228,6 +2342,7 @@ mod tests { .map(parent) .collect(), transactions_commitment: TransactionsCommitment::from_bytes([0x55; 32]), + application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2266,6 +2381,25 @@ mod tests { CandidateCarrierV1::try_new(full_args(committee), committee).unwrap() } + fn application_header( + committee: &Committee, + author: AuthorityIndex, + marker: u8, + ) -> RbcCanonicalHeader { + RbcCanonicalHeader::try_new( + author, + 1, + committee + .authorities() + .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) + .collect(), + Vec::new(), + 0x1122_3344_5566_7700 + u64::from(marker), + TransactionsCommitment::from_bytes([marker; 32]), + ) + .unwrap() + } + #[test] fn canonical_content_and_reference_have_frozen_golden_bytes() { let committee = Committee::new_test(vec![1; 4]); @@ -2318,6 +2452,65 @@ mod tests { ); } + #[test] + fn application_carrier_uses_v2_and_round_trips_the_exact_canonical_header() { + let committee = Committee::new_test(vec![1; 4]); + let application = application_header(&committee, 3, 0xA5); + let mut application_args = args(&committee, 3, 2); + application_args.transactions_commitment = application.transactions_commitment(); + application_args.application_header = Some(application.clone()); + let candidate = CandidateCarrierV1::try_new(application_args.clone(), &committee).unwrap(); + let content = candidate.canonical_content_bytes().unwrap(); + let wire = candidate.canonical_wire_bytes().unwrap(); + assert_eq!(content[1], CARRIER_FORMAT_VERSION_V2); + assert_eq!(wire[1], CARRIER_WIRE_FORMAT_VERSION_V2); + + let decoded_content = + CandidateCarrierV1::decode_content(&content, &committee, Some(candidate.reference())) + .unwrap(); + let decoded_wire = + CandidateCarrierV1::decode_wire(&wire, &committee, Some(candidate.reference())) + .unwrap(); + assert_eq!( + decoded_content.header().application_header(), + Some(&application) + ); + assert_eq!(decoded_wire, decoded_content); + + let changed_application = application_header(&committee, 3, 0xA6); + application_args.transactions_commitment = changed_application.transactions_commitment(); + application_args.application_header = Some(changed_application); + assert_ne!( + CandidateCarrierV1::try_new(application_args, &committee) + .unwrap() + .reference(), + candidate.reference() + ); + } + + #[test] + fn application_carrier_rejects_author_or_commitment_mismatch() { + let committee = Committee::new_test(vec![1; 4]); + let application = application_header(&committee, 3, 0xB1); + let mut bad_commitment = args(&committee, 3, 2); + bad_commitment.application_header = Some(application.clone()); + assert_eq!( + CandidateCarrierV1::try_new(bad_commitment, &committee), + Err(RbcDagError::ApplicationCommitmentMismatch) + ); + + let mut bad_author = args(&committee, 2, 2); + bad_author.transactions_commitment = application.transactions_commitment(); + bad_author.application_header = Some(application); + assert_eq!( + CandidateCarrierV1::try_new(bad_author, &committee), + Err(RbcDagError::ApplicationAuthorMismatch { + carrier: 2, + application: 3, + }) + ); + } + #[test] fn every_canonical_carrier_field_is_bound_to_the_reference() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index 192385aa..88504e35 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -1459,6 +1459,7 @@ mod tests { own_prev, weak_parents, transactions_commitment: TransactionsCommitment::default(), + application_header: None, data_acknowledgments: Vec::new(), phase_batch, consensus_vertex: None, diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 1bda66fe..f0a94206 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -839,6 +839,7 @@ mod tests { own_prev: previous[author as usize], weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: vertex, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 1fcfd07d..a81e0d71 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -18,6 +18,7 @@ use std::{ use crate::{ crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ AuthenticatedCarrierV1, CandidateCarrierV1, CarrierAuthenticationV1, CarrierAuthorizerV1, CarrierHeaderV1Args, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, @@ -122,6 +123,15 @@ pub(crate) struct ShadowOutboundEnvelopeV1 { authentication_sidecar: Vec, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct LocalOutboundMetadataV1 { + pub(crate) round: RoundNumber, + pub(crate) transactions_commitment: TransactionsCommitment, + pub(crate) creation_time_ns: TimestampNs, + pub(crate) control_shape: bool, + pub(crate) application: Option, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ShadowIngressDispositionV1 { Authenticated, @@ -574,6 +584,18 @@ impl StarfishRbcDagShadowV1 { self.model.pending_phase_backlog_len() } + /// Whether the exact phase prefix encodable in the open carrier contains + /// work for an application-bearing carrier. This lets the prototype send + /// application-critical ECHO/READY promptly without turning control-only + /// carrier certification into an unpaced self-sustaining loop. + pub(crate) fn has_pending_application_phase_work(&self) -> bool { + self.model.pending_phase_batch().iter().any(|statement| { + self.candidates + .get(&statement.target()) + .is_some_and(|candidate| candidate.header().application_header().is_some()) + }) + } + pub(crate) fn admitted_reference( &self, authority: AuthorityIndex, @@ -633,6 +655,21 @@ impl StarfishRbcDagShadowV1 { round: RoundNumber, transactions_commitment: TransactionsCommitment, creation_time_ns: TimestampNs, + ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { + self.create_local_carrier_with_application( + round, + transactions_commitment, + None, + creation_time_ns, + ) + } + + fn create_local_carrier_with_application( + &mut self, + round: RoundNumber, + transactions_commitment: TransactionsCommitment, + application_header: Option, + creation_time_ns: TimestampNs, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { self.ensure_live()?; let (own_prev, weak_parents) = self.model.local_parent_set()?; @@ -643,6 +680,7 @@ impl StarfishRbcDagShadowV1 { own_prev, weak_parents, transactions_commitment, + application_header, data_acknowledgments: Vec::new(), phase_batch: self.model.pending_phase_batch(), consensus_vertex: None, @@ -681,6 +719,24 @@ impl StarfishRbcDagShadowV1 { self.create_local_carrier(round, TransactionsCommitment::default(), creation_time_ns) } + /// Assign one exact direct application header to the currently open + /// independent carrier slot. The complete canonical header is committed + /// by the V2 carrier and is therefore recoverable from carrier content. + pub(crate) fn create_local_application_carrier( + &mut self, + application_header: RbcCanonicalHeader, + creation_time_ns: TimestampNs, + ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { + let round = self.model.local_carrier_round(); + let commitment = application_header.transactions_commitment(); + self.create_local_carrier_with_application( + round, + commitment, + Some(application_header), + creation_time_ns, + ) + } + /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -909,7 +965,7 @@ impl StarfishRbcDagShadowV1 { /// payload and creation timestamp before accepting new observations. pub(crate) fn local_outbound_metadata( &self, - ) -> Result, ShadowErrorV1> { + ) -> Result, ShadowErrorV1> { self.journal .snapshot() .retransmissions() @@ -920,13 +976,17 @@ impl StarfishRbcDagShadowV1 { .candidates .get(&reference) .ok_or(ShadowErrorV1::MissingOutboundCandidate(reference))?; - Ok(( - candidate.header().carrier_round(), - candidate.header().transactions_commitment(), - candidate.header().creation_time_ns(), - candidate.header().data_acknowledgments().is_empty() + Ok(LocalOutboundMetadataV1 { + round: candidate.header().carrier_round(), + transactions_commitment: candidate.header().transactions_commitment(), + creation_time_ns: candidate.header().creation_time_ns(), + control_shape: candidate.header().data_acknowledgments().is_empty() && candidate.header().consensus_vertex().is_none(), - )) + application: candidate + .header() + .application_header() + .map(RbcCanonicalHeader::reference), + }) }) .collect() } @@ -950,6 +1010,30 @@ impl StarfishRbcDagShadowV1 { .collect() } + /// Exact application headers whose enclosing carriers reached embedded + /// RBC delivery. Control-only carrier deliveries are intentionally absent. + pub(crate) fn delivered_application_headers( + &self, + ) -> Result, ShadowErrorV1> { + self.delivered + .iter() + .filter_map(|carrier_reference| { + let candidate = match self.candidates.get(carrier_reference) { + Some(candidate) => candidate, + None => { + return Some(Err(ShadowErrorV1::MissingDeliveredCandidate( + *carrier_reference, + ))); + } + }; + candidate + .header() + .application_header() + .map(|header| Ok((*carrier_reference, header.clone()))) + }) + .collect() + } + /// Compare protocol-independent delivery sets. Multiple transaction /// commitments for one `(author, round)` slot make the comparison /// ambiguous instead of being resolved by arrival or reference order. @@ -2850,6 +2934,7 @@ mod tests { own_prev: carrier_genesis_reference(author), weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2886,6 +2971,7 @@ mod tests { transactions_commitment: TransactionsCommitment::from_bytes( [0xD0 + author as u8; 32], ), + application_header: None, data_acknowledgments: Vec::new(), phase_batch: vec![statement], consensus_vertex: None, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 62501be8..0f3b784d 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -97,12 +97,13 @@ impl ShadowServiceModeV1 { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] struct ShadowLocalCarrierV1 { author: AuthorityIndex, round: RoundNumber, transactions_commitment: crate::crypto::TransactionsCommitment, creation_time_ns: TimestampNs, + application_header: RbcCanonicalHeader, } impl ShadowLocalCarrierV1 { @@ -112,6 +113,7 @@ impl ShadowLocalCarrierV1 { round: header.reference().round, transactions_commitment: header.transactions_commitment(), creation_time_ns: header.meta_creation_time_ns(), + application_header: header.clone(), } } } @@ -180,12 +182,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { &self, header: &RbcCanonicalHeader, ) -> Result<(), ShadowServiceErrorV1> { - if self.mode.is_autonomous() { - // The autonomous carrier clock is deliberately independent from - // direct consensus rounds. Direct headers continue through the - // authoritative path and cannot consume carrier slots. - return Ok(()); - } self.send(ShadowServiceMessageV1::LocalCarrier( ShadowLocalCarrierV1::from_direct_header(header), )) @@ -372,6 +368,10 @@ pub(crate) enum ShadowServiceEventV1 { message: NetworkMessage, }, Delivered(ShadowDeliveryIdentityV1), + EmbeddedApplicationDelivered { + carrier: BlockReference, + header: RbcCanonicalHeader, + }, Comparison(ShadowDeliveryComparisonV1), Input { kind: &'static str, @@ -420,7 +420,7 @@ pub(crate) enum ShadowServiceErrorV1 { ConflictingLocalHeader(RoundNumber), MissingRecoveredLocalHeader(RoundNumber), RecoveredLocalHeaderMismatch(RoundNumber), - AutonomousWalContainsApplicationCarrier(RoundNumber), + AutonomousWalContainsInvalidCarrier(RoundNumber), LocalHeaderAuthority { expected: AuthorityIndex, actual: AuthorityIndex, @@ -501,9 +501,9 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "persisted shadow carrier and recovered direct header disagree at round {round}" ), - Self::AutonomousWalContainsApplicationCarrier(round) => write!( + Self::AutonomousWalContainsInvalidCarrier(round) => write!( formatter, - "autonomous carrier-clock WAL contains a non-heartbeat local carrier at round {round}" + "autonomous carrier-clock WAL contains an invalid local carrier at round {round}" ), Self::LocalHeaderAuthority { expected, actual } => write!( formatter, @@ -593,6 +593,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( own_authority: AuthorityIndex, context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, heartbeat_interval: Duration, wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< @@ -612,7 +613,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( own_authority, context, authorizer, - Vec::new(), + recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, wal_sync_policy, ) @@ -649,9 +650,10 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( actual: local.author, }); } - if let Some(previous) = pending_local.insert(local.round, local) { + let round = local.round; + if let Some(previous) = pending_local.insert(round, local.clone()) { if previous != local { - return Err(ShadowServiceErrorV1::ConflictingLocalHeader(local.round)); + return Err(ShadowServiceErrorV1::ConflictingLocalHeader(round)); } } } @@ -759,8 +761,16 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let persisted_local = match core.local_outbound_metadata() { Ok(metadata) => metadata .into_iter() - .map(|(round, commitment, creation_time_ns, control_shape)| { - (round, (commitment, creation_time_ns, control_shape)) + .map(|metadata| { + ( + metadata.round, + ( + metadata.transactions_commitment, + metadata.creation_time_ns, + metadata.control_shape, + metadata.application, + ), + ) }) .collect::>(), Err(error) => { @@ -773,29 +783,80 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( return; } }; + let mut assigned_applications = BTreeSet::new(); let durable_round = core.local_carrier_round(); if mode.is_autonomous() { - if let Some((round, _)) = - persisted_local - .iter() - .find(|(_, (commitment, _, control_shape))| { - *commitment != crate::crypto::TransactionsCommitment::default() - || !*control_shape - }) - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::AutonomousWalContainsApplicationCarrier( - *round, - ) - .to_string(), - }) - .await; - return; + for (carrier_round, (commitment, _, control_shape, application)) in &persisted_local { + if !*control_shape { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::AutonomousWalContainsInvalidCarrier( + *carrier_round, + ) + .to_string(), + }) + .await; + return; + } + let Some(application) = application else { + if *commitment != crate::crypto::TransactionsCommitment::default() { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::AutonomousWalContainsInvalidCarrier( + *carrier_round, + ) + .to_string(), + }) + .await; + return; + } + continue; + }; + let Some(recovered) = pending_local.get(&application.round) else { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::MissingRecoveredLocalHeader( + application.round, + ) + .to_string(), + }) + .await; + return; + }; + if recovered.application_header.reference() != *application + || recovered.transactions_commitment != *commitment + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch( + application.round, + ) + .to_string(), + }) + .await; + return; + } + assigned_applications.insert(*application); } + pending_local.retain(|_, local| { + !assigned_applications.contains(&local.application_header.reference()) + }); } else { - for (round, (commitment, creation_time_ns, _)) in &persisted_local { + for (round, (commitment, creation_time_ns, _, application)) in &persisted_local { + if application.is_some() { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) + .to_string(), + }) + .await; + return; + } let Some(recovered) = pending_local.get(round) else { let _ = startup_events .send(ShadowServiceEventV1::Rejected { @@ -833,6 +894,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( .await; return; } + pending_local.retain(|round, _| *round >= core.local_carrier_round()); } let reported_shadow_deliveries = match core.delivered_identities() { Ok(identities) => identities.into_iter().collect::>(), @@ -847,12 +909,26 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }; let recovered_shadow_deliveries = reported_shadow_deliveries.clone(); + let reported_application_deliveries = match core.delivered_application_headers() { + Ok(headers) => headers + .into_iter() + .map(|(_, header)| header.reference()) + .collect(), + Err(error) => { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: error.to_string(), + }) + .await; + return; + } + }; let reported_shadow_delivery_slots = reported_shadow_deliveries .iter() .map(delivery_slot) .collect(); let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); - pending_local.retain(|round, _| *round >= core.local_carrier_round()); let sync_round = core.local_carrier_round(); let state = ShadowServiceStateV1 { core, @@ -867,6 +943,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( observed_topology: BTreeMap::new(), invalidated_by_overload: actor_invalidated_by_overload, pending_local, + assigned_applications, pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), @@ -879,6 +956,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( heartbeat_notification_pending: actor_heartbeat_notification_pending, direct_deliveries: BTreeSet::new(), reported_shadow_deliveries, + reported_application_deliveries, recovered_shadow_deliveries, comparison_backlog, reported_matches: BTreeSet::new(), @@ -993,6 +1071,7 @@ struct ShadowServiceStateV1 { observed_topology: BTreeMap, invalidated_by_overload: Arc>>, pending_local: BTreeMap, + assigned_applications: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, @@ -1005,6 +1084,7 @@ struct ShadowServiceStateV1 { heartbeat_notification_pending: Arc, direct_deliveries: BTreeSet, reported_shadow_deliveries: BTreeSet, + reported_application_deliveries: BTreeSet, recovered_shadow_deliveries: BTreeSet, comparison_backlog: ShadowComparisonBacklogV1, reported_matches: BTreeSet, @@ -1214,7 +1294,7 @@ impl ShadowServiceStateV1 { self.emit_clock_state(); } - fn try_create_autonomous_heartbeat(&mut self) { + fn try_create_autonomous_carrier(&mut self) { if !self.mode.is_autonomous() || !self.core.can_create_carrier() { self.emit_clock_state(); return; @@ -1226,10 +1306,27 @@ impl ShadowServiceStateV1 { .try_into() .unwrap_or(TimestampNs::MAX); let before = self.core.wal_counts(); - match self.core.create_local_control_heartbeat(creation_time_ns) { + let application_round = self.pending_local.keys().next().copied(); + let application = application_round.and_then(|round| self.pending_local.remove(&round)); + let result = match &application { + Some(application) => self.core.create_local_application_carrier( + application.application_header.clone(), + creation_time_ns, + ), + None => self.core.create_local_control_heartbeat(creation_time_ns), + }; + match result { Ok((envelope, effects)) => { + if let Some(application) = application { + self.assigned_applications + .insert(application.application_header.reference()); + } self.emit(ShadowServiceEventV1::Input { - kind: "heartbeat", + kind: if application_round.is_some() { + "application_carrier" + } else { + "heartbeat" + }, outcome: "accepted", }); self.report_wal_delta(before); @@ -1237,6 +1334,9 @@ impl ShadowServiceStateV1 { self.process_effects(effects); } Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { + if let Some(application) = application { + self.pending_local.insert(application.round, application); + } // The local slot is open syntactically but cannot yet name a // quorum of exact previous-round admitted parents. A later // authenticated ingress or timer tick retries it. @@ -1246,7 +1346,12 @@ impl ShadowServiceStateV1 { }); self.emit_clock_state(); } - Err(error) => self.mark_fatal(error), + Err(error) => { + if let Some(application) = application { + self.pending_local.insert(application.round, application); + } + self.mark_fatal(error); + } } } @@ -1258,7 +1363,7 @@ impl ShadowServiceStateV1 { fn drive_autonomous_catch_up(&mut self) { while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_heartbeat(); + self.try_create_autonomous_carrier(); if self.core.local_carrier_round() == round_before { break; } @@ -1286,6 +1391,37 @@ impl ShadowServiceStateV1 { ); return; } + if self.mode.is_autonomous() { + let application_reference = local.application_header.reference(); + if self.assigned_applications.contains(&application_reference) { + self.emit(ShadowServiceEventV1::Input { + kind: "application", + outcome: "already_assigned", + }); + return; + } + if let Some(existing) = self.pending_local.get(&local.round) { + if existing == &local { + self.emit(ShadowServiceEventV1::Input { + kind: "application", + outcome: "duplicate", + }); + } else { + self.reject( + None, + ShadowServiceErrorV1::ConflictingLocalHeader(local.round), + ); + } + return; + } + self.pending_local.insert(local.round, local); + self.emit(ShadowServiceEventV1::Input { + kind: "application", + outcome: "queued", + }); + self.retry_pending_local(); + return; + } let durable_round = self.core.local_carrier_round(); if local.round < durable_round { self.emit(ShadowServiceEventV1::Input { @@ -1324,6 +1460,19 @@ impl ShadowServiceStateV1 { /// advances the model; recovered historical headers below that clock are /// harmless idempotent replays. fn retry_pending_local(&mut self) { + if self.mode.is_autonomous() { + while (!self.pending_local.is_empty() || self.core.has_pending_application_phase_work()) + && self.core.can_create_carrier() + && !self.fatal + { + let round_before = self.core.local_carrier_round(); + self.try_create_autonomous_carrier(); + if self.core.local_carrier_round() == round_before { + break; + } + } + return; + } loop { let durable_round = self.core.local_carrier_round(); self.pending_local @@ -1590,6 +1739,7 @@ impl ShadowServiceStateV1 { } self.report_wal_delta(before); self.process_effects(outcome.effects().to_vec()); + self.retry_pending_local(); if self .core .admitted_reference(response.author, response.round) @@ -1643,6 +1793,21 @@ impl ShadowServiceStateV1 { self.emit_slot_comparison(slot); self.emit_comparison_backlog(); } + let applications = match self.core.delivered_application_headers() { + Ok(applications) => applications, + Err(error) => { + self.reject(None, error); + return; + } + }; + for (carrier, header) in applications { + if self + .reported_application_deliveries + .insert(header.reference()) + { + self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header }); + } + } } fn emit_slot_comparison(&mut self, slot: ShadowDeliverySlotV1) { @@ -1716,10 +1881,9 @@ fn run_shadow_service( }); state.emit_comparison_backlog(); state.process_effects(open_report.recovery_effects().to_vec()); + state.retry_pending_local(); if state.mode.is_autonomous() { state.emit_clock_state(); - } else { - state.retry_pending_local(); } } @@ -1914,7 +2078,7 @@ fn run_shadow_service( state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); } - ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_heartbeat(), + ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(), ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); @@ -2113,6 +2277,7 @@ mod tests { authority, self.context, ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), + Vec::new(), heartbeat_interval, wal_sync_policy, ) @@ -2202,6 +2367,7 @@ mod tests { events: &mut [mpsc::Receiver], open_rounds: &mut [RoundNumber], deliveries: &mut [usize], + application_deliveries: &mut [BTreeSet], sync_requests: &mut usize, target_open_round: RoundNumber, ) { @@ -2255,6 +2421,12 @@ mod tests { ShadowServiceEventV1::Delivered(_) => { deliveries[sender] = deliveries[sender].saturating_add(1); } + ShadowServiceEventV1::EmbeddedApplicationDelivered { + header, + .. + } => { + application_deliveries[sender].insert(header.reference()); + } ShadowServiceEventV1::Rejected { error, .. } if error.contains("FutureCarrierOutsideBuffer") || error.contains("unexpected shadow response") => {} @@ -2423,6 +2595,7 @@ mod tests { own_prev: carrier_genesis_reference(author), weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2511,6 +2684,7 @@ mod tests { let mut open_rounds = vec![1; n]; let mut deliveries = vec![0; n]; + let mut application_deliveries = vec![BTreeSet::new(); n]; let mut sync_requests = 0; for (authority, handle) in handles.iter().enumerate() { for peer in 0..n { @@ -2528,6 +2702,7 @@ mod tests { &mut events, &mut open_rounds, &mut deliveries, + &mut application_deliveries, &mut sync_requests, fixed_round + 1, ) @@ -2553,6 +2728,100 @@ mod tests { } } + #[tokio::test] + async fn autonomous_application_header_wins_the_open_carrier_slot_and_uses_v2() { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start_autonomous(0); + wait_ready(&mut events).await; + let application = direct_header(0, 1, 0x6A); + handle.local_header(&application).unwrap(); + + let envelope = next_carrier(&mut events, 1).await; + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &envelope.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + assert_eq!(candidate.header().carrier_round(), 1); + assert_eq!(candidate.header().application_header(), Some(&application)); + assert_eq!( + envelope.canonical_carrier[1], + crate::starfish_rbc_dag::CARRIER_WIRE_FORMAT_VERSION_V2 + ); + + handle.shutdown().await.unwrap(); + task.await.unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn embedded_rbc_delivers_every_application_header_from_the_carrier_dag() { + let harness = Harness::new(); + let mut handles = Vec::new(); + let mut events = Vec::new(); + let mut tasks = Vec::new(); + for authority in 0..N as AuthorityIndex { + let (handle, mut node_events, task) = harness.start_autonomous(authority); + wait_ready(&mut node_events).await; + handles.push(handle); + events.push(node_events); + tasks.push(task); + } + for (authority, handle) in handles.iter().enumerate() { + for peer in 0..N { + if peer != authority { + handle.peer_connected(peer as AuthorityIndex).unwrap(); + } + } + } + + let applications = (0..N as AuthorityIndex) + .map(|authority| direct_header(authority, 1, 0x70 + authority as u8)) + .collect::>(); + let expected = applications + .iter() + .map(RbcCanonicalHeader::reference) + .collect::>(); + for (handle, application) in handles.iter().zip(&applications) { + handle.local_header(application).unwrap(); + } + + let mut open_rounds = vec![1; N]; + let mut deliveries = vec![0; N]; + let mut application_deliveries = vec![BTreeSet::new(); N]; + let mut sync_requests = 0; + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut application_deliveries, + &mut sync_requests, + 5, + ) + .await; + + assert!( + application_deliveries + .iter() + .all(|delivered| delivered == &expected), + "every node must deliver every exact embedded application: {application_deliveries:?}" + ); + assert!( + open_rounds.iter().all(|round| *round >= 5), + "application-critical phase carriers must not wait for a heartbeat tick" + ); + assert_eq!(sync_requests, 0); + + drop(events); + for handle in &handles { + handle.shutdown().await.unwrap(); + } + for task in tasks { + task.await.unwrap(); + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn four_node_autonomous_zero_load_clock_delivers_mature_heartbeats() { assert_autonomous_zero_load_progress(4).await; @@ -2694,6 +2963,7 @@ mod tests { } let mut open_rounds = vec![1; N]; let mut deliveries = vec![0; N]; + let mut application_deliveries = vec![BTreeSet::new(); N]; let mut sync_requests = 0; for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); @@ -2703,6 +2973,7 @@ mod tests { &mut events, &mut open_rounds, &mut deliveries, + &mut application_deliveries, &mut sync_requests, 2, ) @@ -2753,6 +3024,7 @@ mod tests { &mut events, &mut open_rounds, &mut deliveries, + &mut application_deliveries, &mut sync_requests, 10, ) @@ -2829,6 +3101,86 @@ mod tests { stop(restarted, restarted_events, restarted_task).await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn autonomous_wal_restart_reconciles_and_replays_exact_application_origin() { + let harness = Harness::new(); + let application = direct_header(0, 1, 0x7B); + let (handle, mut events, task) = harness.start_autonomous(0); + wait_ready(&mut events).await; + handle.local_header(&application).unwrap(); + let original = next_carrier(&mut events, 1).await; + stop(handle, events, task).await; + + let (restarted, mut restarted_events, restarted_task) = + start_starfish_rbc_dag_autonomous_clock_service_v1( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + vec![application.clone()], + Duration::from_secs(60 * 60), + ShadowWalSyncPolicyV1::EveryBatch, + ) + .unwrap(); + loop { + match next_event(&mut restarted_events).await { + ShadowServiceEventV1::Ready { autonomous_clock } => { + assert!(autonomous_clock); + break; + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("application WAL restart failed: {error}") + } + _ => {} + } + } + restarted.local_header(&application).unwrap(); + restarted + .carrier_sync_request( + 1, + RbcDagShadowCarrierSyncRequest { + author: 0, + round: 1, + }, + ) + .unwrap(); + loop { + if let ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrierSyncResponse(response), + } = next_event(&mut restarted_events).await + { + assert_eq!(response.canonical_carrier, original.canonical_carrier); + assert_eq!( + response.authentication_sidecar, + original.authentication_sidecar + ); + break; + } + } + stop(restarted, restarted_events, restarted_task).await; + + let (_invalid, invalid_events, invalid_task) = + start_starfish_rbc_dag_autonomous_clock_service_v1( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + Vec::new(), + Duration::from_secs(60 * 60), + ShadowWalSyncPolicyV1::EveryBatch, + ) + .unwrap(); + assert!( + startup_rejection(invalid_events) + .await + .contains("no matching recovered direct header") + ); + invalid_task.await.unwrap(); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn buffered_wal_reports_append_without_durability_and_reopens_after_clean_shutdown() { let harness = Harness::new(); @@ -2899,6 +3251,7 @@ mod tests { transactions_commitment: TransactionsCommitment::from_bytes( [0xB0 + author as u8; 32], ), + application_header: None, data_acknowledgments: Vec::new(), phase_batch: vec![statement], consensus_vertex: None, diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 30dbf87b..721be73a 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -198,6 +198,12 @@ pub(crate) struct RbcServiceHandle { sender: mpsc::UnboundedSender, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RbcPhaseAuthorityV1 { + Direct, + EmbeddedCarrierDag, +} + impl RbcServiceHandle { #[allow(dead_code)] pub(crate) async fn start_local_header( @@ -314,6 +320,7 @@ impl RbcServiceHandle { } /// Start the single Starfish-RBC state-machine owner. +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub(crate) fn start_starfish_rbc_service( committee: Arc, @@ -331,6 +338,37 @@ pub(crate) fn start_starfish_rbc_service( JoinHandle<()>, ), RbcServiceError, +> { + start_starfish_rbc_service_with_phase_authority( + committee, + own_authority, + protocol_instance, + initial_authentication, + mac_keys, + initial_authenticator, + local_round, + header_retry_interval, + RbcPhaseAuthorityV1::Direct, + ) +} + +pub(crate) fn start_starfish_rbc_service_with_phase_authority( + committee: Arc, + own_authority: AuthorityIndex, + protocol_instance: RbcProtocolInstanceId, + initial_authentication: BlockAuthenticationScheme, + mac_keys: Arc>, + initial_authenticator: RbcInitialAuthenticator, + local_round: RoundNumber, + header_retry_interval: Duration, + phase_authority: RbcPhaseAuthorityV1, +) -> Result< + ( + RbcServiceHandle, + mpsc::UnboundedReceiver, + JoinHandle<()>, + ), + RbcServiceError, > { if header_retry_interval.is_zero() { return Err(RbcServiceError::ZeroHeaderRetryInterval); @@ -364,6 +402,7 @@ pub(crate) fn start_starfish_rbc_service( staged_notifications: AHashSet::new(), retained_initials: BTreeMap::new(), retained_phases: BTreeSet::new(), + phase_authority, }; let task = tokio::spawn(run_service(state, message_rx, header_retry_interval)); Ok((RbcServiceHandle { sender: message_tx }, event_rx, task)) @@ -437,6 +476,7 @@ struct RbcServiceState { /// Authorized local phase intents. Tags are rematerialized for the peer /// on replay rather than retaining or cloning a tagged wire message. retained_phases: BTreeSet<(BlockReference, RbcPhase)>, + phase_authority: RbcPhaseAuthorityV1, } impl RbcServiceState { @@ -454,9 +494,11 @@ impl RbcServiceState { self.accept_direct_initial(peer, proposal); } RbcServiceMessage::Phase { peer, message } => { - match self.kernel.handle_phase(peer, message) { - Ok(effects) => self.process_effects(effects), - Err(error) => self.reject(Some(peer), error.into()), + if self.phase_authority == RbcPhaseAuthorityV1::Direct { + match self.kernel.handle_phase(peer, message) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(peer), error.into()), + } } } RbcServiceMessage::HeaderRequest { peer, block_ref } => { @@ -708,6 +750,9 @@ impl RbcServiceState { for effect in effects { match effect { RbcEffect::MulticastPhase { phase, block_ref } => { + if self.phase_authority == RbcPhaseAuthorityV1::EmbeddedCarrierDag { + continue; + } self.retained_phases.insert((block_ref, phase)); let recipients: Vec<_> = self .committee @@ -727,6 +772,9 @@ impl RbcServiceState { self.note_pending_fetch(block_ref, holders); } RbcEffect::Deliver(header) => { + if self.phase_authority == RbcPhaseAuthorityV1::EmbeddedCarrierDag { + continue; + } self.pending_fetches.remove(&header.reference()); let _ = self.events.send(RbcServiceEvent::Delivered(header)); } @@ -922,6 +970,27 @@ mod tests { .unwrap() } + fn start_embedded_phase_service() -> ( + RbcServiceHandle, + mpsc::UnboundedReceiver, + JoinHandle<()>, + ) { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + start_starfish_rbc_service_with_phase_authority( + committee, + 0, + instance(), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + RbcInitialAuthenticator::Mac, + 1, + Duration::from_secs(3_600), + RbcPhaseAuthorityV1::EmbeddedCarrierDag, + ) + .unwrap() + } + async fn next_event(events: &mut mpsc::UnboundedReceiver) -> RbcServiceEvent { tokio::time::timeout(Duration::from_secs(2), events.recv()) .await @@ -996,6 +1065,37 @@ mod tests { } } + #[tokio::test] + async fn embedded_phase_authority_emits_init_but_no_direct_echo_or_delivery() { + let (handle, mut events, task) = start_embedded_phase_service(); + let canonical = handle.start_local_header(local_header(1, 4)).await.unwrap(); + let mut staged = false; + let mut initials = 0; + for _ in 0..4 { + match next_event(&mut events).await { + RbcServiceEvent::HeaderStaged(header) => { + assert_eq!(header.reference(), canonical.reference()); + staged = true; + } + RbcServiceEvent::Network { + message: NetworkMessage::RbcInitial(_), + .. + } => initials += 1, + event => panic!("unexpected init-only event: {event:?}"), + } + } + assert!(staged); + assert_eq!(initials, 3); + assert!( + tokio::time::timeout(Duration::from_millis(25), events.recv()) + .await + .is_err(), + "direct ECHO/READY or delivery escaped the embedded authority boundary" + ); + drop(handle); + task.await.unwrap(); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blocking_local_start_waits_for_kernel_selection_and_event_enqueue() { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 0b8d6a50..c53364e6 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -59,19 +59,19 @@ impl Validator { "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" )); } - if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { + if public_config + .parameters + .starfish_rbc_dag_embedded_rbc_authority + && (!public_config.parameters.starfish_rbc_dag_shadow + || !public_config.parameters.starfish_rbc_dag_autonomous_clock) + { return Err(eyre!( - "Starfish-RBC-DAG autonomous clock requires consensus 'starfish-rbc'" + "Starfish-RBC-DAG embedded RBC authority requires the autonomous RBC-DAG shadow" )); } - if public_config.parameters.starfish_rbc_dag_autonomous_clock - && public_config - .parameters - .starfish_rbc_dag_heartbeat_interval_ms - == 0 - { + if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { return Err(eyre!( - "Starfish-RBC-DAG autonomous heartbeat interval must be greater than zero" + "Starfish-RBC-DAG autonomous clock requires consensus 'starfish-rbc'" )); } if public_config.parameters.starfish_rbc_dag_shadow && !is_starfish_rbc { @@ -490,41 +490,6 @@ mod smoke_tests { })); } - #[tokio::test] - async fn autonomous_clock_rejects_zero_heartbeat_interval() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_shadow = true; - public_config.parameters.starfish_rbc_dag_autonomous_clock = true; - public_config - .parameters - .starfish_rbc_dag_heartbeat_interval_ms = 0; - public_config - .parameters - .refresh_starfish_rbc_protocol_instance(); - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish-rbc".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("autonomous heartbeat interval must be greater than zero") - })); - } - async fn run_commit_test( consensus: &str, block_authentication: Option<&str>, @@ -545,6 +510,7 @@ mod smoke_tests { port_offset, starfish_rbc_dag_shadow, false, + false, ) .await; } @@ -555,6 +521,7 @@ mod smoke_tests { port_offset: u16, starfish_rbc_dag_shadow: bool, autonomous_clock: bool, + embedded_rbc_authority: bool, ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); @@ -563,17 +530,20 @@ mod smoke_tests { public_config.parameters.block_authentication = block_authentication.map(str::to_string); public_config.parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; public_config.parameters.starfish_rbc_dag_autonomous_clock = autonomous_clock; - if autonomous_clock { - public_config - .parameters - .starfish_rbc_dag_heartbeat_interval_ms = 50; - } + public_config + .parameters + .starfish_rbc_dag_embedded_rbc_authority = embedded_rbc_authority; if consensus == "starfish-rbc" { public_config .parameters .refresh_starfish_rbc_protocol_instance(); } - let parameters = Parameters::default(); + let mut parameters = Parameters::default(); + if autonomous_clock { + // Exercise the shared Starfish/carrier pacemaker contract without + // making the integration test wait for production timeouts. + parameters.leader_timeout = Some(Duration::from_millis(50)); + } let dir = TempDir::new().unwrap(); let private_configs = NodePrivateConfig::new_for_benchmarks(dir.as_ref(), committee_size); @@ -636,6 +606,12 @@ mod smoke_tests { .with_label_values(&["delivery", "shadow"]) .get() > 0 + && (!embedded_rbc_authority + || metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .get() + > 0) && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 }) { break; @@ -655,6 +631,32 @@ mod smoke_tests { 0, "autonomous mode must not claim direct-round comparison" ); + if embedded_rbc_authority { + assert!( + metrics + .network_message_bytes_sent_total + .with_label_values(&["rbc_initial"]) + .get() + > 0, + "direct INIT remains the application/payload transport" + ); + assert_eq!( + metrics + .network_message_bytes_sent_total + .with_label_values(&["rbc_echo"]) + .get(), + 0, + "direct RBC ECHO must be disabled under embedded authority" + ); + assert_eq!( + metrics + .network_message_bytes_sent_total + .with_label_values(&["rbc_ready"]) + .get(), + 0, + "direct RBC READY must be disabled under embedded authority" + ); + } } } else if starfish_rbc_dag_shadow { let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR @@ -800,7 +802,13 @@ mod smoke_tests { #[tokio::test] async fn starfish_rbc_dag_autonomous_clock_advances_without_owning_consensus() { - run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1700, true, true).await; + run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1700, true, true, false) + .await; + } + + #[tokio::test] + async fn starfish_rbc_dag_embedded_rbc_is_the_only_phase_authority() { + run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1740, true, true, true).await; } #[tokio::test] diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index b22cbc30..c593b25c 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -80,9 +80,10 @@ enum Operation { /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_heartbeat_interval_ms: Option, + /// Let embedded carrier ECHO/READY delivery certify application + /// headers. Requires the autonomous RBC-DAG mode. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_embedded_rbc_authority: bool, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -124,9 +125,10 @@ enum Operation { /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_heartbeat_interval_ms: Option, + /// Let embedded carrier ECHO/READY delivery certify application + /// headers. Requires the autonomous RBC-DAG mode. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_embedded_rbc_authority: bool, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -190,9 +192,10 @@ enum Operation { /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_heartbeat_interval_ms: Option, + /// Let embedded carrier ECHO/READY delivery certify application + /// headers. Requires the autonomous RBC-DAG mode. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_embedded_rbc_authority: bool, /// Benchmark-only: write ordered shadow-WAL frames but force them to /// stable storage only at clean shutdown. This run is not crash-safe. #[clap(long, default_value_t = false)] @@ -233,7 +236,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_embedded_rbc_authority, } => { run( authority, @@ -246,7 +249,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_embedded_rbc_authority, ) .await? } @@ -263,7 +266,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_embedded_rbc_authority, data_dir, base_ip, storage_backend, @@ -285,7 +288,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_embedded_rbc_authority, data_dir, base_ip, storage_backend, @@ -309,7 +312,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_dag_shadow_buffered_wal, duration_secs, dissemination_mode, @@ -323,11 +326,10 @@ async fn main() -> Result<()> { node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; + node_parameters.starfish_rbc_dag_embedded_rbc_authority = + starfish_rbc_dag_embedded_rbc_authority; node_parameters.starfish_rbc_dag_shadow_buffered_wal = starfish_rbc_dag_shadow_buffered_wal; - if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; - } if consensus_protocol == "starfish-rbc" { node_parameters.refresh_starfish_rbc_protocol_instance(); } @@ -451,6 +453,12 @@ async fn local_benchmark( } ); } + if node_parameters.starfish_rbc_dag_autonomous_clock { + println!( + "Carrier idle timeout: {} ms (shared Starfish leader pacemaker)", + node_parameters.leader_timeout.as_millis() + ); + } if let Some(latency) = node_parameters.uniform_latency_ms { println!("Network Latency: {latency} ms (uniform)"); } else { @@ -489,6 +497,8 @@ async fn local_benchmark( let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; let starfish_rbc_dag_autonomous_clock_expected = node_parameters.starfish_rbc_dag_autonomous_clock; + let starfish_rbc_dag_embedded_rbc_authority_expected = + node_parameters.starfish_rbc_dag_embedded_rbc_authority; // Create temporary directories for each validator let base_dir = PathBuf::from("local-benchmark"); @@ -661,6 +671,7 @@ async fn local_benchmark( committee_size, starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, + starfish_rbc_dag_embedded_rbc_authority_expected, autonomous_clock_baselines.clone(), Some(counter_baselines.clone()), ); @@ -689,6 +700,7 @@ async fn local_benchmark( committee_size, starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, + starfish_rbc_dag_embedded_rbc_authority_expected, autonomous_clock_baselines, Some(counter_baselines), ); @@ -710,7 +722,7 @@ async fn run( block_authentication: Option, starfish_rbc_dag_shadow: bool, starfish_rbc_dag_autonomous_clock: bool, - starfish_rbc_dag_heartbeat_interval_ms: Option, + starfish_rbc_dag_embedded_rbc_authority: bool, ) -> Result<()> { tracing::info!("Starting node {authority}"); @@ -728,10 +740,10 @@ async fn run( if starfish_rbc_dag_autonomous_clock { public_config.parameters.starfish_rbc_dag_autonomous_clock = true; } - if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { + if starfish_rbc_dag_embedded_rbc_authority { public_config .parameters - .starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + .starfish_rbc_dag_embedded_rbc_authority = true; } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" @@ -772,7 +784,7 @@ async fn dryrun( block_authentication: Option, starfish_rbc_dag_shadow: bool, starfish_rbc_dag_autonomous_clock: bool, - starfish_rbc_dag_heartbeat_interval_ms: Option, + starfish_rbc_dag_embedded_rbc_authority: bool, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -817,9 +829,8 @@ async fn dryrun( node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; - if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; - } + node_parameters.starfish_rbc_dag_embedded_rbc_authority = + starfish_rbc_dag_embedded_rbc_authority; ensure_starfish_rbc_protocol_instance(&consensus_protocol, &mut node_parameters); if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; @@ -1001,8 +1012,7 @@ mod tests { "mac", "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", - "--starfish-rbc-dag-heartbeat-interval-ms", - "125", + "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-dag-shadow-buffered-wal", ]) .unwrap(); @@ -1012,7 +1022,7 @@ mod tests { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, + starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_dag_shadow_buffered_wal, .. } = args.operation @@ -1023,7 +1033,7 @@ mod tests { assert_eq!(block_authentication.as_deref(), Some("mac")); assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); - assert_eq!(starfish_rbc_dag_heartbeat_interval_ms, Some(125)); + assert!(starfish_rbc_dag_embedded_rbc_authority); assert!(starfish_rbc_dag_shadow_buffered_wal); } @@ -1032,7 +1042,6 @@ mod tests { let mut parameters = NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }; @@ -1045,6 +1054,5 @@ mod tests { ); assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); - assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 4edae30e..764467bd 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,17 +1,18 @@ # Starfish-RBC-DAG protocol design -Status: milestone-four persisted, non-authoritative optimistic carrier-clock shadow; no -authoritative protocol, safety/liveness, or performance claim +Status: milestone-five authoritative embedded-RBC prototype; consensus projection and end-to-end +safety/liveness proof remain incomplete The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not -implemented. The milestone-three direct-header comparison runtime is enabled with `--consensus -starfish-rbc --starfish-rbc-dag-shadow`. Milestone four adds a separate control-only runtime with -`--starfish-rbc-dag-autonomous-clock`; its carrier rounds advance independently through -authenticated admission and empty heartbeats. Performance experiments may add -`--starfish-rbc-dag-shadow-buffered-wal` to remove per-transition disk synchronization; that -profile is explicitly not crash-safe. Both modes leave the direct prototype's DAG, -pacemaker, commit, and output unchanged. The eventual protocol is new, not a transport option or a -version-two alias for `starfish-rbc`. +implemented. The staged prototype runs under `starfish-rbc`: direct-header comparison uses +`--starfish-rbc-dag-shadow`, the independent carrier clock adds +`--starfish-rbc-dag-autonomous-clock`, and milestone five makes embedded carrier ECHO/READY the +only application-header certification authority with +`--starfish-rbc-dag-embedded-rbc-authority`. Direct INIT still transports the application payload, +but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add +`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Consensus +projection, commit, and output still use the existing Starfish DAG. The eventual protocol is new, +not a transport option or a version-two alias for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -46,20 +47,24 @@ carrier/RBC, certified-projection, decision, and crash-journal models. Milestone opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired direct/shadow delivery observations. Milestone four adds an independently authenticated autonomous heartbeat namespace, sequential `Q`-admitted carrier clock, bounded future buffering, exact-slot -synchronization, and clock-state metrics. The direct `starfish-rbc` service remains the only -authority: shadow admission, delivery, recovery, clock advancement, or failure cannot advance a -proposal, mark a DAG vertex clean, vote, commit, or order output. - -Autonomous mode is intentionally control-only at this milestone. It ignores direct application -headers and uses a distinct WAL and authentication protocol instance. This avoids falsely equating -direct consensus rounds with faster carrier rounds, but also means milestone four does not yet -measure application latency through the new carrier DAG. Application-origin assignment and the -certified consensus projection remain later milestones. +synchronization, and clock-state metrics. Milestone five adds a version-two application carrier +containing the exact canonical application header, durable application-origin reconciliation, +immediate application/phase scheduling, and an opt-in authority boundary that prevents direct +ECHO/READY from certifying a header. Idle control heartbeats use the same resolved leader timeout as +the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application carriers and their +encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. + +The current authoritative mode changes only header certification. Consensus vertices, certified +projection, commits, and application ordering remain on the existing Starfish DAG. Direct INIT is +still the application-payload transport and is not a certification vote. Replacing that remaining +wrapper with a payload-only path and moving consensus into the clean carrier projection are later +milestones. Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode -requires an identical recovered direct-header history, while autonomous mode reopens its -control-only heartbeat history without direct headers and serves byte-identical exact-slot -responses. This is not a full validator crash-recovery claim. The authoritative direct +requires an identical recovered direct-header history, control-only autonomous history reopens +without direct headers, and autonomous application origins must match recovered direct history. +Every mode serves byte-identical exact-slot responses. This is not a full validator crash-recovery +claim. The authoritative direct `starfish-rbc` baseline does not yet durably record its remote-slot ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest @@ -908,15 +913,18 @@ cannot be used for crash-safety claims. Benchmark output reports appended and du separately. The clone-based reducer remains intentionally unoptimized until measurement shows it matters. -A matched 10-validator local A/B on 2026-08-11 used a full 60-second active transaction window, -the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous -heartbeat. The harness waits through generator warmup, snapshots cumulative counters at the active -boundary, and drains final latency samples. +A matched 10-validator local sequence on 2026-08-11 used a full 60-second active transaction +window, the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and the buffered +benchmark WAL. Milestone-five idle carriers use the same resolved 600 ms Push leader timeout as +Starfish-RBC; application and encodable phase carriers are immediate. The harness waits through +generator warmup, snapshots cumulative counters at the active boundary, and drains final latency +samples. | Profile | Verdict | TPS | p50 block | p50 E2E | Outbound | |---|---:|---:|---:|---:|---:| | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | | Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | +| Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | | Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted @@ -924,8 +932,16 @@ heartbeats, 27,180 embedded-RBC deliveries, 27,424 appended batches, zero round pending recovery. Its latency and throughput match the shadow-off baseline while making the expected extra carrier traffic visible. The crash-safe run shed shadow work and is reported only as a diagnostic: it isolates synchronous persistence as a severe observer effect and must not be -cited as a protocol result. Neither run measures application latency through the carrier DAG yet, -because the direct Starfish-RBC path remains authoritative in milestone four. +cited as a protocol result. + +The milestone-five run delivered 10,722 application headers through embedded RBC, reached carrier +rounds 458–459, and ended with zero pending recovery. Its direct ECHO and READY paths were disabled; +the composed four-validator test also asserts zero such outbound messages. Its higher latency is +not evidence for a bad 250 ms timer—the independent timer was removed, and the same Starfish +timeout was used. It exposes the transitional clean-predecessor gate: the existing direct Starfish +DAG still serializes proposal creation on embedded delivery. Milestone six must replace that gate +with the optimistic carrier clock plus certified consensus projection before the complete protocol +can be expected to recover Starfish pipelining. ## 19. Contained implementation milestones @@ -947,10 +963,11 @@ Every milestone is committed separately. 4. **Optimistic carrier clock (implemented, opt-in control shadow):** run a separately namespaced, control-only heartbeat carrier plane with the distinct authenticated-admission latch, sequential quorum clock, bounded future buffer, exact-slot synchronization, durable restart, and clock - validity metrics while consensus still uses the current direct baseline. Application headers are - not assigned to autonomous carrier rounds yet. -5. **Authoritative embedded RBC:** remove direct ECHO/READY authority only after shadow tests show - identical delivery under reordering, loss, equivocation, poisoned tags, and restart. + validity metrics while consensus still uses the current direct baseline. +5. **Authoritative embedded RBC (implemented, opt-in):** encode exact canonical application headers + in version-two carriers, durably reconcile their origins, schedule application/phase carriers + immediately, and remove direct ECHO/READY/delivery authority. Direct INIT remains payload + transport; composed tests assert zero direct ECHO/READY traffic and positive embedded delivery. 6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit leader choice, contiguous delivery frontiers, and strict clean-only committer consumers. 7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct @@ -967,8 +984,8 @@ the executable model or measured prototype: - production maximum future-carrier buffer and payload runahead (the executable model deliberately uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); -- the production control-heartbeat rate under low load and backpressure (the autonomous shadow's - configurable 250 ms default is an empirical test value, not a protocol constant); +- whether the shared Starfish leader-timeout policy needs a separately proved adaptive low-load + rule; the prototype intentionally does not introduce a second heartbeat timeout; - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and From 2275c063623e382b5fac8b3eb9043cc49a4331ed Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:14:16 +0200 Subject: [PATCH 34/62] Add certified RBC-DAG consensus projection --- README.md | 28 +- crates/orchestrator/src/main.rs | 8 +- crates/orchestrator/src/measurements.rs | 37 +- crates/starfish-core/src/config.rs | 11 +- crates/starfish-core/src/metrics.rs | 81 ++- crates/starfish-core/src/net_sync.rs | 74 ++- .../src/starfish_rbc_dag/model.rs | 25 +- .../src/starfish_rbc_dag/projection.rs | 31 +- .../src/starfish_rbc_dag_shadow.rs | 520 ++++++++++++++++-- .../src/starfish_rbc_dag_shadow_service.rs | 159 +++++- crates/starfish-core/src/syncer.rs | 9 +- crates/starfish-core/src/validator.rs | 45 +- crates/starfish/src/main.rs | 24 +- docs/starfish-rbc-dag-protocol.md | 47 +- 14 files changed, 969 insertions(+), 130 deletions(-) diff --git a/README.md b/README.md index 30f59c49..ea0b261c 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ prototype with the limitations documented in its [protocol specification](docs/s **Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate logical projection. Its canonical types, deterministic models, crash journal, comparison shadow, autonomous carrier clock, -and opt-in authoritative embedded-RBC path are implemented. Run the comparison shadow with +authoritative embedded-RBC path, and certified logical consensus projection are implemented. Run +the comparison shadow with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; add `--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact application headers in version-two carriers and make embedded ECHO/READY/delivery their sole @@ -59,9 +60,11 @@ certification authority. Direct INIT remains payload transport, but direct ECHO/ blocks in that mode. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for Starfish-RBC by default); application and encodable phase carriers are emitted immediately. -The current milestone changes certification, not consensus: the existing Starfish DAG still -consumes the embedded deliveries and retains its clean-predecessor proposal gate. The certified -carrier projection and frontier linearizer are the next milestones. Shadow traffic shares the +Autonomous carriers now embed durably locked consensus vertices with quorum strong parents, +explicit Vote/NoVote choices, and exact delivery frontiers. Only RBC-delivered, data-available, +prefix-closed vertices enter the projection or its leader decisions. The existing Starfish DAG is +still the temporary application-output scaffold; the committed frontier linearizer is the next +milestone. Shadow traffic shares the validator's network socket and bandwidth, and deployment requires a homogeneous new-binary committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; `--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean @@ -74,8 +77,9 @@ production retains a short embedded-RBC pipeline tail, so benchmark validation u unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between the cumulative direct and shadow delivery counters. Autonomous runs instead require `starfish_rbc_dag_shadow_clock_valid == 1`, heartbeat/WAL progress, advancing carrier rounds, -in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports -at most 60 validators in mirror mode and 20 in autonomous mode. +in-window embedded-RBC delivery, projected-vertex and clean projected-commit progress, and bounded +clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 +in autonomous mode. A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared @@ -86,14 +90,22 @@ nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Sta | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | | Autonomous comparison, direct RBC authoritative | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | +| Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | The milestone-five run produced 10,722 embedded application deliveries, reached carrier rounds 458–459, and ended with zero pending recovery. It also proves that the earlier 250 ms experimental heartbeat was not the latency cause: application and phase carriers are already event-driven, and using the shared 600 ms timeout did not restore the direct baseline. The remaining slowdown is an expected warning about the transitional architecture—the old direct DAG still serializes proposal -creation on embedded RBC cleanliness. Do not tune that obsolete gate; milestone six must let the -optimistic carrier clock advance independently and feed only certified vertices into consensus. +creation on embedded RBC cleanliness. Milestone six now lets the optimistic carrier clock advance +independently and feeds only certified vertices into the logical committer; milestone seven must +remove the remaining legacy output gate by committing deterministic frontier deltas. +The milestone-six run reached carrier rounds 356–359 with 35,506 carrier deliveries, 8,059 +application deliveries, 8,487 projected vertices, 830 clean direct commits, and zero pending +recovery. Its further latency increase is a structural red flag, not a projection-speed claim: +certified decisions currently run alongside the old clean-predecessor/output path, so the benchmark +still pays for both. The next measurement is meaningful only after milestone seven removes that +legacy gate. The local harness starts its timer after transaction-generator warmup, subtracts warmup counters, and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 761eb399..e145c583 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -63,13 +63,13 @@ pub struct Opts { #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, - /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative - /// shadow. + /// Enable the persisted Starfish-RBC-DAG research runtime. Without the + /// autonomous flag it remains comparison-only. #[clap(long, global = true)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + /// Run the independent Starfish-RBC-DAG carrier clock and certified + /// projection. Requires `--starfish-rbc-dag-shadow`. #[clap(long, global = true)] starfish_rbc_dag_autonomous_clock: bool, diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index e63c0d0f..600aab66 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -258,13 +258,16 @@ impl Measurement { x.as_str(), "starfish_rbc_dag_shadow_inputs_total" | "starfish_rbc_dag_shadow_delivery_comparisons_total" + | "starfish_rbc_dag_projection_decisions_total" ) => { match sample.value { prometheus_parse::Value::Counter(value) => { - let shadow_bucket = if x - == "starfish_rbc_dag_shadow_delivery_comparisons_total" - { + let shadow_bucket = if matches!( + x.as_str(), + "starfish_rbc_dag_shadow_delivery_comparisons_total" + | "starfish_rbc_dag_projection_decisions_total" + ) { sample .labels .get("outcome") @@ -291,6 +294,7 @@ impl Measurement { | "starfish_rbc_dag_shadow_wal_durable_batches_total" | "starfish_rbc_dag_shadow_wal_durable_records_total" | "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total" + | "starfish_rbc_dag_projected_vertices_total" ) => { match sample.value { @@ -1159,6 +1163,13 @@ impl MeasurementsCollection { ) && self.scalar_counter_increased( "starfish_rbc_dag_shadow_wal_appended_records_total", *scraper_id, + ) && self.scalar_counter_increased( + "starfish_rbc_dag_projected_vertices_total", + *scraper_id, + ) && self.count_bucket_increased( + "starfish_rbc_dag_projection_decisions_total", + *scraper_id, + "direct_commit", ) && self.scalar_gauge_increased( "starfish_rbc_dag_shadow_carrier_round", *scraper_id, @@ -1637,6 +1648,26 @@ mod test { ..Measurement::default() }, ); + collection.add( + scraper_id, + "starfish_rbc_dag_projected_vertices_total".to_owned(), + Measurement { + timestamp, + count: wal_durable_records, + scalar: wal_durable_records as f64, + ..Measurement::default() + }, + ); + collection.add( + scraper_id, + "starfish_rbc_dag_projection_decisions_total".to_owned(), + Measurement { + timestamp, + count_buckets: HashMap::from([("direct_commit".to_owned(), wal_durable_records)]), + count: wal_durable_records, + ..Measurement::default() + }, + ); collection.add( scraper_id, "starfish_rbc_dag_shadow_pending_recovery".to_owned(), diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 8b7de0b0..d0c5aebe 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -67,14 +67,13 @@ pub struct NodeParameters { /// other protocols. #[serde(default)] pub starfish_rbc_protocol_instance: Option<[u8; 32]>, - /// Run the persisted Starfish-RBC-DAG carrier implementation alongside - /// the authoritative direct Starfish-RBC service. Shadow delivery is - /// observational only and cannot affect the DAG, pacemaker, or commits. + /// Run the persisted Starfish-RBC-DAG implementation alongside the + /// legacy direct Starfish-RBC service. Without autonomous mode it is a + /// comparison-only mirror. #[serde(default)] pub starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create an autonomous - /// optimistic carrier clock. This remains experimental and requires - /// `starfish_rbc_dag_shadow`. + /// Run an independent carrier clock and certified logical projection. + /// This remains experimental and requires `starfish_rbc_dag_shadow`. #[serde(default)] pub starfish_rbc_dag_autonomous_clock: bool, /// Use embedded carrier ECHO/READY delivery as the certification authority diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 5d47fc66..f99b0f4f 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -41,8 +41,8 @@ pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR: i64 = 4; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; -/// Local-benchmark guards for the non-authoritative autonomous carrier -/// clock. The round-skew limit matches the executable model's bounded future +/// Local-benchmark guards for the autonomous RBC-DAG carrier clock. The +/// round-skew limit matches the executable model's bounded future /// buffer. A healthy clock can transiently retain phase work, but its carrier /// capacity exceeds the two RBC statements generated per admitted value; a /// sixteen-committee backlog therefore leaves generous scheduling headroom @@ -198,6 +198,8 @@ pub struct Metrics { pub starfish_rbc_dag_shadow_admitted_authors: IntGauge, pub starfish_rbc_dag_shadow_admitted_stake: IntGauge, pub starfish_rbc_dag_shadow_buffered_authenticated: IntGauge, + pub starfish_rbc_dag_projected_vertices_total: IntCounter, + pub starfish_rbc_dag_projection_decisions_total: IntCounterVec, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -268,6 +270,8 @@ pub struct AutonomousClockBenchmarkBaseline { accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, + projected_vertices: u64, + projection_decisions: u64, wal_batches: u64, wal_records: u64, carrier_round: i64, @@ -293,6 +297,8 @@ struct AutonomousClockBenchmarkSummary { accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, + projected_vertices: u64, + projection_decisions: u64, wal_batches: u64, wal_records: u64, pending_recovery: i64, @@ -347,6 +353,13 @@ fn summarize_autonomous_clock_benchmark( .with_label_values(&["delivery", "embedded_application"]) .get() > baseline.delivered_applications) + && metrics.starfish_rbc_dag_projected_vertices_total.get() + > baseline.projected_vertices + && metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get() + > baseline.projection_decisions && metrics .starfish_rbc_dag_shadow_wal_appended_batches_total .get() @@ -403,6 +416,19 @@ fn summarize_autonomous_clock_benchmark( .get() }) .sum(); + let projected_vertices = metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_projected_vertices_total.get()) + .sum(); + let projection_decisions = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get() + }) + .sum(); let wal_batches = metrics .iter() .map(|metrics| { @@ -470,6 +496,8 @@ fn summarize_autonomous_clock_benchmark( accepted_heartbeats, delivered_carriers, delivered_applications, + projected_vertices, + projection_decisions, wal_batches, wal_records, pending_recovery, @@ -510,6 +538,11 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get(), + projected_vertices: self.starfish_rbc_dag_projected_vertices_total.get(), + projection_decisions: self + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get(), wal_batches: self .starfish_rbc_dag_shadow_wal_appended_batches_total .get(), @@ -936,7 +969,7 @@ impl Metrics { .unwrap(), starfish_rbc_dag_shadow_clock_valid: register_int_gauge_with_registry!( "starfish_rbc_dag_shadow_clock_valid", - "State of the non-authoritative autonomous carrier clock (1 valid, 0 disabled/invalid, -1 starting)", + "State of the autonomous RBC-DAG carrier clock and projection runtime (1 valid, 0 disabled/invalid, -1 starting)", registry, ) .unwrap(), @@ -970,6 +1003,20 @@ impl Metrics { registry, ) .unwrap(), + starfish_rbc_dag_projected_vertices_total: register_int_counter_with_registry!( + "starfish_rbc_dag_projected_vertices_total", + "RBC-delivered, data-available consensus vertices admitted to the certified carrier projection", + registry, + ) + .unwrap(), + starfish_rbc_dag_projection_decisions_total: + register_int_counter_vec_with_registry!( + "starfish_rbc_dag_projection_decisions_total", + "Clean-only Starfish leader decisions produced by the certified carrier projection", + &["outcome"], + registry, + ) + .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1712,10 +1759,12 @@ impl Metrics { table.add_row(row![ b->"Clock/WAL progress:", format!( - "heartbeats={}, carrier deliveries={}, application deliveries={}, WAL batches={}, records={}, open rounds={}..{}", + "heartbeats={}, carrier deliveries={}, application deliveries={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, summary.delivered_carriers, summary.delivered_applications, + summary.projected_vertices, + summary.projection_decisions, summary.wal_batches, summary.wal_records, summary.minimum_round, @@ -2232,6 +2281,11 @@ mod tests { metrics .starfish_rbc_dag_shadow_buffered_authenticated .set(2); + metrics.starfish_rbc_dag_projected_vertices_total.inc(); + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .inc(); let gathered = registry.gather(); for name in [ @@ -2254,6 +2308,8 @@ mod tests { "starfish_rbc_dag_shadow_admitted_authors", "starfish_rbc_dag_shadow_admitted_stake", "starfish_rbc_dag_shadow_buffered_authenticated", + "starfish_rbc_dag_projected_vertices_total", + "starfish_rbc_dag_projection_decisions_total", ] { assert!( gathered.iter().any(|family| family.get_name() == name), @@ -2299,6 +2355,11 @@ mod tests { metrics .starfish_rbc_dag_shadow_buffered_authenticated .set(buffered_authenticated); + metrics.starfish_rbc_dag_projected_vertices_total.inc(); + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .inc(); metrics } @@ -2363,6 +2424,11 @@ mod tests { .starfish_rbc_dag_shadow_wal_durable_records_total .inc(); metrics.starfish_rbc_dag_shadow_carrier_round.inc(); + metrics.starfish_rbc_dag_projected_vertices_total.inc(); + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .inc(); } assert!( @@ -2373,7 +2439,7 @@ mod tests { #[test] fn embedded_authority_summary_requires_application_delivery_progress() { - let metrics = vec![autonomous_clock_metrics(8, 0, 0)]; + let metrics = [autonomous_clock_metrics(8, 0, 0)]; let baselines = metrics .iter() .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) @@ -2394,6 +2460,11 @@ mod tests { .starfish_rbc_dag_shadow_wal_appended_records_total .inc(); metrics.starfish_rbc_dag_shadow_carrier_round.inc(); + metrics.starfish_rbc_dag_projected_vertices_total.inc(); + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .inc(); assert!( !summarize_autonomous_clock_benchmark( diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 74dc8e4b..f1c37fa9 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -48,7 +48,7 @@ use crate::{ starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader, RbcCommitteeId, RbcProtocolInstanceId}, starfish_rbc_dag::{ RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, - storage::ShadowWalSyncPolicyV1, + projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, @@ -1815,10 +1815,11 @@ impl NetworkSyncer Ok(headers) => Some(headers), Err(error) => { // A partial local history would make delivery comparisons - // meaningless. Disable the whole observational run while - // allowing authoritative direct RBC to continue. + // meaningless in mirror mode and make autonomous local + // application-origin reconciliation unsafe. Disable the + // RBC-DAG runtime while allowing the legacy path to continue. tracing::error!( - "Disabling non-authoritative RBC-DAG shadow because recovered direct headers cannot be reconciled: {error}" + "Disabling RBC-DAG runtime because recovered direct headers cannot be reconciled: {error}" ); None } @@ -1975,9 +1976,7 @@ impl NetworkSyncer Ok((service, events, task)) => (Some(service), Some(events), Some(task)), Err(error) => { invalidate_shadow_run(&metrics); - tracing::error!( - "Disabling non-authoritative Starfish-RBC-DAG shadow: {error}" - ); + tracing::error!("Disabling Starfish-RBC-DAG runtime: {error}"); (None, None, None) } } @@ -2165,6 +2164,17 @@ impl NetworkSyncer .syncer .add_transaction_data(vec![item], DataSource::StarfishRbcPayload) .await; + if let Some(ref shadow) = + event_inner.starfish_rbc_dag_shadow_service + { + if let Err(error) = shadow.application_data_available(block_ref) { + invalidate_shadow_run(&rbc_metrics); + tracing::warn!( + ?block_ref, + "Failed to record RBC-DAG application availability: {error}" + ); + } + } } RbcServiceEvent::Delivered(header) => { if let Some(ref shadow) = @@ -2296,6 +2306,26 @@ impl NetworkSyncer } } } + ShadowServiceEventV1::VertexProjected(reference) => { + shadow_metrics + .starfish_rbc_dag_projected_vertices_total + .inc(); + tracing::debug!(?reference, "RBC-DAG consensus vertex projected"); + } + ShadowServiceEventV1::LeaderDecided(decision) => { + let outcome = match decision { + ProjectionDecisionV1::DirectCommit { .. } => "direct_commit", + ProjectionDecisionV1::DirectSkip { .. } => "direct_skip", + ProjectionDecisionV1::IndirectCommit { .. } => "indirect_commit", + ProjectionDecisionV1::IndirectSkip { .. } => "indirect_skip", + ProjectionDecisionV1::Undecided { .. } => "undecided", + }; + shadow_metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&[outcome]) + .inc(); + tracing::debug!(?decision, "RBC-DAG projected leader decided"); + } ShadowServiceEventV1::ComparisonBacklog { unpaired_direct, unpaired_shadow, @@ -2417,7 +2447,7 @@ impl NetworkSyncer shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } tracing::warn!( - "Rejected non-authoritative RBC-DAG shadow input from {:?}: {}", + "Rejected RBC-DAG runtime input from {:?}: {}", peer, error ); @@ -2428,23 +2458,39 @@ impl NetworkSyncer }); // Start bridge task that forwards reconstructed transaction data to core + let bridge_metrics = metrics.clone(); let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); + let bridge_metrics = bridge_metrics.clone(); handle.spawn(async move { while let Some(items) = decoded_rx.recv().await { // Reconstruction proves we now have the shard data for the // entire batch. - let shard_refs = items.iter().map(|item| item.block_reference).collect(); + let shard_refs = items + .iter() + .map(|item| item.block_reference) + .collect::>(); bridge_inner .cordial_knowledge .send(CordialKnowledgeMessage::DagParts { headers: Vec::new(), - shards: shard_refs, + shards: shard_refs.clone(), }); bridge_inner .syncer .add_transaction_data(items, DataSource::ShardReconstructor) .await; + if let Some(ref shadow) = bridge_inner.starfish_rbc_dag_shadow_service { + for reference in shard_refs { + if let Err(error) = shadow.application_data_available(reference) { + invalidate_shadow_run(&bridge_metrics); + tracing::warn!( + ?reference, + "Failed to record reconstructed RBC-DAG availability: {error}" + ); + } + } + } } }) }); @@ -2799,13 +2845,13 @@ impl NetworkSyncer .await { Ok(Ok(())) => {} - Ok(Err(error)) => tracing::warn!( - "Non-authoritative RBC-DAG shadow did not acknowledge shutdown: {error}" - ), + Ok(Err(error)) => { + tracing::warn!("RBC-DAG runtime did not acknowledge shutdown: {error}") + } Err(_) => { shadow_shutdown_timed_out = true; tracing::warn!( - "Timed out stopping non-authoritative RBC-DAG shadow; detaching it from validator shutdown" + "Timed out stopping RBC-DAG runtime; detaching it from validator shutdown" ); if let Some(task) = rbc_dag_shadow_service_task.as_ref() { task.abort(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index 88504e35..d4f0e64f 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -24,7 +24,7 @@ use crate::{ }; use super::{ - AuthenticatedCarrierV1, CandidateCarrierV1, LocallyAuthenticatedCarrierV1, + AuthenticatedCarrierV1, CandidateCarrierV1, LeaderChoiceV1, LocallyAuthenticatedCarrierV1, MAX_PHASE_STATEMENTS_V1, RbcDagCommitteeId, RbcDagContextV1, RbcPhaseStatementV1, carrier_genesis_reference, }; @@ -91,6 +91,19 @@ pub enum ModelTraceEvent { }, /// The local author fixed one exact carrier before authorizing its ECHO. LocalCarrierFixed(BlockReference), + /// The optional consensus vertex became the author's immutable value for + /// its logical consensus round. This follows fixing the enclosing carrier + /// and precedes any outbound exposure. + ConsensusSlotLocked { + consensus_round: RoundNumber, + enclosing_carrier: BlockReference, + }, + /// The local Vote/NoVote choice embedded in the fixed consensus vertex + /// became immutable before the carrier can be exposed. + LeaderChoiceLocked { + consensus_round: RoundNumber, + choice: LeaderChoiceV1, + }, /// Bracha delivery became slot-global and immutable. DeliveryLocked(BlockReference), /// Existing non-durable output retained in its exact reducer order. @@ -662,6 +675,16 @@ impl RbcDagModel { self.preflight_receive(&carrier)?; self.own_fixed.insert(round, reference); log.proof(ModelTraceEvent::LocalCarrierFixed(reference)); + if let Some(vertex) = header.consensus_vertex() { + log.proof(ModelTraceEvent::ConsensusSlotLocked { + consensus_round: vertex.consensus_round(), + enclosing_carrier: reference, + }); + log.proof(ModelTraceEvent::LeaderChoiceLocked { + consensus_round: vertex.consensus_round(), + choice: vertex.leader_choice(), + }); + } for statement in &expected_phase_batch { self.pending_phase_set.remove(statement); } diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index f0a94206..21797741 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -35,7 +35,7 @@ pub struct LeaderSlotV1 { pub round: RoundNumber, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum ProjectionDecisionV1 { DirectCommit { leader: ConsensusVertexReference, @@ -254,6 +254,35 @@ impl CertifiedProjectionModel { self.vertices.contains_key(&reference) } + /// Deterministic clean values at one logical consensus round. Byzantine + /// equivocations remain visible as distinct exact references. + pub fn projected_values_at_round(&self, round: RoundNumber) -> Vec { + self.vertices_at_round(round) + .map(|(reference, _)| reference) + .collect() + } + + pub fn projected_vertex( + &self, + reference: ConsensusVertexReference, + ) -> Option<&ConsensusVertexV1> { + self.vertices + .get(&reference) + .map(|projected| &projected.vertex) + } + + /// Current exact closed carrier-prefix frontier in authority order. + pub fn closed_frontier(&self) -> DeliveryFrontierV1 { + self.committee + .authorities() + .map(|authority| self.closed_tip(authority)) + .collect() + } + + pub fn projected_vertex_count(&self) -> usize { + self.vertices.len() + } + pub fn slot_values( &self, author: AuthorityIndex, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index a81e0d71..e12441e7 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -1,13 +1,13 @@ // Copyright (c) 2026 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! Durable, single-owner shadow execution for the embedded-RBC Starfish DAG. +//! Durable, single-owner execution for the embedded-RBC Starfish DAG. //! -//! This adapter is deliberately non-authoritative: it consumes the same -//! carrier bytes as the live protocol, persists its own deterministic input -//! and trace log, and reports comparison results without influencing the live -//! protocol. One [`ShadowWalV1`] batch is one reducer transition. Effects are -//! returned only after that complete batch has reached durable storage. +//! Direct-mirror mode remains observational. Autonomous mode owns carrier +//! admission, embedded RBC, and certified projection decisions; the legacy +//! Starfish DAG is still the temporary application-output scaffold until M7. +//! One [`ShadowWalV1`] batch is one reducer transition. Effects are returned +//! only after that complete batch has reached durable storage. use std::{ collections::{BTreeMap, BTreeSet}, @@ -21,13 +21,17 @@ use crate::{ starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ AuthenticatedCarrierV1, CandidateCarrierV1, CarrierAuthenticationV1, CarrierAuthorizerV1, - CarrierHeaderV1Args, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, - RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, + CarrierHeaderV1Args, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, + LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagError, + RbcPhaseStatementV1, carrier_genesis_reference, journal::{ IngressProvenanceV1, JournalErrorV1, JournalEventV1, ValidatedJournalBatchV1, WriteAheadJournalV1, }, model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, + projection::{ + CertifiedProjectionError, CertifiedProjectionModel, LeaderSlotV1, ProjectionDecisionV1, + }, storage::{ MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, ShadowWalSummaryV1, ShadowWalSyncPolicyV1, ShadowWalV1, @@ -47,6 +51,7 @@ const RECORD_AUTHENTICATED_INGRESS: u8 = 0x01; const RECORD_CANDIDATE_RETENTION: u8 = 0x02; const RECORD_CANDIDATE_RECOVERY: u8 = 0x03; const RECORD_LOCAL_OUTBOUND_CONTENT: u8 = 0x04; +const RECORD_DATA_AVAILABLE: u8 = 0x05; const RECORD_MODEL_TRACE: u8 = 0x10; const RECORD_LOCAL_OUTBOUND_SIDECAR: u8 = 0x11; const RECORD_LOCAL_OUTBOUND_EXPOSE: u8 = 0x12; @@ -58,6 +63,8 @@ const TRACE_PHASE_CURSOR_ADVANCED: u8 = 0x03; const TRACE_LOCAL_CARRIER_FIXED: u8 = 0x04; const TRACE_DELIVERY_LOCKED: u8 = 0x05; const TRACE_EFFECT: u8 = 0x06; +const TRACE_CONSENSUS_SLOT_LOCKED: u8 = 0x07; +const TRACE_LEADER_CHOICE_LOCKED: u8 = 0x08; const EFFECT_NEED_CARRIER: u8 = 0x00; const EFFECT_DELIVERED: u8 = 0x01; @@ -259,6 +266,7 @@ pub(crate) enum ShadowCodecErrorV1 { TrailingBytes(usize), InvalidProvenance(u8), InvalidPhase(u8), + InvalidLeaderChoice(u8), InvalidTrace(u8), InvalidEffect(u8), NonCanonicalHolders, @@ -309,6 +317,7 @@ pub(crate) enum ShadowErrorV1 { MissingDeliveredCandidate(BlockReference), PostDurabilityCommit(ModelError), PostDurabilityJournal(JournalErrorV1), + Projection(CertifiedProjectionError), Poisoned, } @@ -381,6 +390,7 @@ impl fmt::Display for ShadowErrorV1 { formatter, "shadow journal commit failed after WAL durability: {error}" ), + Self::Projection(error) => write!(formatter, "{error}"), Self::Poisoned => formatter.write_str("shadow core is poisoned"), } } @@ -394,6 +404,7 @@ impl Error for ShadowErrorV1 { Self::Carrier(error) => Some(error), Self::Model(error) | Self::PostDurabilityCommit(error) => Some(error), Self::Journal(error) | Self::PostDurabilityJournal(error) => Some(error), + Self::Projection(error) => Some(error), _ => None, } } @@ -429,6 +440,12 @@ impl From for ShadowErrorV1 { } } +impl From for ShadowErrorV1 { + fn from(error: CertifiedProjectionError) -> Self { + Self::Projection(error) + } +} + #[derive(Clone)] enum ShadowInputV1 { AuthenticatedIngress { @@ -438,6 +455,7 @@ enum ShadowInputV1 { CandidateRetention(CandidateCarrierV1), CandidateRecovery(CandidateCarrierV1), LocalOutbound(LocallyAuthenticatedCarrierV1), + DataAvailable(BlockReference), } impl ShadowInputV1 { @@ -455,14 +473,18 @@ impl ShadowInputV1 { Self::LocalOutbound(authenticated) => { ModelInputRecord::LocalCarrierFixed(authenticated.clone()) } + Self::DataAvailable(reference) => ModelInputRecord::DataAvailable(*reference), } } - fn candidate(&self) -> &CandidateCarrierV1 { + fn candidate(&self) -> Option<&CandidateCarrierV1> { match self { - Self::AuthenticatedIngress { authenticated, .. } => authenticated.candidate(), - Self::CandidateRetention(candidate) | Self::CandidateRecovery(candidate) => candidate, - Self::LocalOutbound(authenticated) => authenticated.candidate(), + Self::AuthenticatedIngress { authenticated, .. } => Some(authenticated.candidate()), + Self::CandidateRetention(candidate) | Self::CandidateRecovery(candidate) => { + Some(candidate) + } + Self::LocalOutbound(authenticated) => Some(authenticated.candidate()), + Self::DataAvailable(_) => None, } } @@ -476,7 +498,7 @@ struct DecodedRawRecord { payload: Vec, } -/// Synchronous, non-authoritative shadow core. +/// Synchronous, durable RBC-DAG core. /// /// This type has one mutable model, journal, and WAL handle and intentionally /// offers no shared-state wrapper. A caller may move it between threads but @@ -495,6 +517,11 @@ pub(crate) struct StarfishRbcDagShadowV1 { ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, requested_recoveries: BTreeMap>, + projection: CertifiedProjectionModel, + projection_rejected: BTreeMap, + projected_decisions: BTreeSet, + pending_projected_vertices: Vec, + pending_projection_decisions: Vec, poisoned: bool, } @@ -532,6 +559,7 @@ impl StarfishRbcDagShadowV1 { let discarded_tail_bytes = recovery.discarded_tail_bytes(); let model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; + let projection = CertifiedProjectionModel::from_committee_context(committee.clone()); let journal = WriteAheadJournalV1::new(context, own_authority); let mut core = Self { committee, @@ -547,6 +575,11 @@ impl StarfishRbcDagShadowV1 { ordinarily_retained_slots: BTreeMap::new(), slot_candidates: BTreeMap::new(), requested_recoveries: BTreeMap::new(), + projection, + projection_rejected: BTreeMap::new(), + projected_decisions: BTreeSet::new(), + pending_projected_vertices: Vec::new(), + pending_projection_decisions: Vec::new(), poisoned: false, }; @@ -554,6 +587,10 @@ impl StarfishRbcDagShadowV1 { let input = core.decode_batch(batch.records())?; core.apply_replayed(input, batch.records(), batch.sequence())?; } + // Historical decisions are reconstructed to validate replay, but are + // not re-emitted as fresh runtime observations after restart. + core.pending_projection_decisions.clear(); + core.pending_projected_vertices.clear(); let recovery_effects = core .requested_recoveries .iter() @@ -635,6 +672,65 @@ impl StarfishRbcDagShadowV1 { .count() } + pub(crate) fn drain_projection_decisions(&mut self) -> Vec { + std::mem::take(&mut self.pending_projection_decisions) + } + + pub(crate) fn drain_projected_vertices(&mut self) -> Vec { + std::mem::take(&mut self.pending_projected_vertices) + } + + /// Persist the external data-availability predicate for one exact + /// application-bearing carrier. Control carriers are available by shape + /// and never require this oracle. + pub(crate) fn mark_carrier_data_available( + &mut self, + reference: BlockReference, + ) -> Result, ShadowErrorV1> { + self.ensure_live()?; + if self + .model + .lifecycle(&reference) + .is_some_and(|lifecycle| lifecycle.data_available) + { + return Ok(Vec::new()); + } + self.apply_durable(ShadowInputV1::DataAvailable(reference)) + } + + pub(crate) fn application_carriers(&self, application: BlockReference) -> Vec { + self.candidates + .iter() + .filter_map(|(reference, candidate)| { + candidate + .header() + .application_header() + .is_some_and(|header| header.reference() == application) + .then_some(*reference) + }) + .collect() + } + + /// Application headers with the canonical empty commitment need no + /// transaction reconstruction. Their exact carrier bytes are therefore + /// sufficient data-availability evidence once the carrier is retained. + pub(crate) fn intrinsically_available_applications(&self) -> Vec { + self.candidates + .values() + .filter_map(|candidate| candidate.header().application_header()) + .filter(|header| header.transactions_commitment() == TransactionsCommitment::default()) + .map(RbcCanonicalHeader::reference) + .collect::>() + .into_iter() + .collect() + } + + pub(crate) fn carrier_data_available(&self, reference: BlockReference) -> bool { + self.model + .lifecycle(&reference) + .is_some_and(|lifecycle| lifecycle.data_available) + } + pub(crate) fn wal_counts(&self) -> (u64, u64) { (self.wal.batch_count(), self.wal.record_count()) } @@ -660,6 +756,7 @@ impl StarfishRbcDagShadowV1 { round, transactions_commitment, None, + None, creation_time_ns, ) } @@ -669,6 +766,7 @@ impl StarfishRbcDagShadowV1 { round: RoundNumber, transactions_commitment: TransactionsCommitment, application_header: Option, + consensus_vertex: Option, creation_time_ns: TimestampNs, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { self.ensure_live()?; @@ -683,7 +781,7 @@ impl StarfishRbcDagShadowV1 { application_header, data_acknowledgments: Vec::new(), phase_batch: self.model.pending_phase_batch(), - consensus_vertex: None, + consensus_vertex, creation_time_ns, }, &self.committee, @@ -714,9 +812,18 @@ impl StarfishRbcDagShadowV1 { pub(crate) fn create_local_control_heartbeat( &mut self, creation_time_ns: TimestampNs, + allow_no_vote: bool, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { let round = self.model.local_carrier_round(); - self.create_local_carrier(round, TransactionsCommitment::default(), creation_time_ns) + let (own_prev, _) = self.model.local_parent_set()?; + let consensus_vertex = self.build_local_consensus_vertex(own_prev, allow_no_vote); + self.create_local_carrier_with_application( + round, + TransactionsCommitment::default(), + None, + consensus_vertex, + creation_time_ns, + ) } /// Assign one exact direct application header to the currently open @@ -726,17 +833,99 @@ impl StarfishRbcDagShadowV1 { &mut self, application_header: RbcCanonicalHeader, creation_time_ns: TimestampNs, + allow_no_vote: bool, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { let round = self.model.local_carrier_round(); let commitment = application_header.transactions_commitment(); + let (own_prev, _) = self.model.local_parent_set()?; + let consensus_vertex = self.build_local_consensus_vertex(own_prev, allow_no_vote); self.create_local_carrier_with_application( round, commitment, Some(application_header), + consensus_vertex, creation_time_ns, ) } + fn build_local_consensus_vertex( + &self, + own_prev: BlockReference, + allow_no_vote: bool, + ) -> Option { + let consensus_round = self.next_local_consensus_round(); + let (strong_parents, leader_choice) = if consensus_round == 1 { + let strong_parents = self + .committee + .committee() + .authorities() + .map(|authority| { + ConsensusVertexReference::new(carrier_genesis_reference(authority), 0) + }) + .collect::>(); + let leader_author = self.committee.committee().elect_leader(0); + ( + strong_parents, + LeaderChoiceV1::Vote { + leader: ConsensusVertexReference::new( + carrier_genesis_reference(leader_author), + 0, + ), + }, + ) + } else { + let parent_round = consensus_round - 1; + let mut by_author = BTreeMap::new(); + for parent in self.projection.projected_values_at_round(parent_round) { + by_author.entry(parent.author()).or_insert(parent); + } + let own_parent = *by_author.get(&self.own_authority)?; + let stake = by_author + .keys() + .filter_map(|authority| self.committee.committee().get_stake(*authority)) + .fold(0, Stake::saturating_add); + if stake < self.committee.committee().quorum_threshold() { + return None; + } + let leader_author = self.committee.committee().elect_leader(parent_round); + let leader_choice = match by_author.get(&leader_author).copied() { + Some(leader) => LeaderChoiceV1::Vote { leader }, + None if allow_no_vote => LeaderChoiceV1::NoVote { + leader_author, + leader_round: parent_round, + }, + None => return None, + }; + let strong_parents = by_author.into_values().collect::>(); + debug_assert!(strong_parents.contains(&own_parent)); + (strong_parents, leader_choice) + }; + + let mut delivery_frontier = self.projection.closed_frontier(); + let own_entry = (own_prev.round != 0).then_some(own_prev); + // The enclosing carrier is not clean yet, so its immediate physical + // predecessor may be ahead of today's closed tip. The immutable + // declaration names that exact predecessor; projection later proves + // the intervening self-chain closed before accepting this vertex. + delivery_frontier.get(self.own_authority as usize)?; + delivery_frontier[self.own_authority as usize] = own_entry; + Some(ConsensusVertexV1::new( + consensus_round, + strong_parents, + delivery_frontier, + leader_choice, + )) + } + + fn next_local_consensus_round(&self) -> RoundNumber { + let snapshot = self.journal.snapshot(); + let mut round: RoundNumber = 1; + while snapshot.consensus_slot(round).is_some() { + round = round.saturating_add(1); + } + round + } + /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -980,8 +1169,7 @@ impl StarfishRbcDagShadowV1 { round: candidate.header().carrier_round(), transactions_commitment: candidate.header().transactions_commitment(), creation_time_ns: candidate.header().creation_time_ns(), - control_shape: candidate.header().data_acknowledgments().is_empty() - && candidate.header().consensus_vertex().is_none(), + control_shape: candidate.header().data_acknowledgments().is_empty(), application: candidate .header() .application_header() @@ -1069,6 +1257,77 @@ impl StarfishRbcDagShadowV1 { Ok(self.wal.shutdown()?) } + fn drive_certified_projection(&mut self) { + loop { + let mut advanced = false; + let candidates = self + .candidates + .iter() + .filter_map(|(reference, candidate)| { + candidate + .header() + .consensus_vertex() + .is_some() + .then_some(*reference) + }) + .collect::>(); + for reference in candidates { + let consensus_round = self + .candidates + .get(&reference) + .and_then(|candidate| candidate.header().consensus_vertex()) + .expect("candidate list contains only consensus vertices") + .consensus_round(); + let vertex_reference = ConsensusVertexReference::new(reference, consensus_round); + if self.projection.is_projected(vertex_reference) + || self.projection_rejected.contains_key(&reference) + { + continue; + } + match self.projection.try_project(reference) { + Ok(projected) => { + self.pending_projected_vertices.push(projected); + advanced = true; + } + Err(error) if projection_error_is_pending(&error) => {} + Err(error) => { + self.projection_rejected.insert(reference, error); + } + } + } + if !advanced { + break; + } + } + + let highest_round = self + .candidates + .values() + .filter_map(|candidate| candidate.header().consensus_vertex()) + .map(ConsensusVertexV1::consensus_round) + .max() + .unwrap_or_default(); + for round in 1..=highest_round.saturating_sub(2) { + let slot = self.projection.leader_slot(round); + if self + .projected_decisions + .iter() + .any(|decision| projection_decision_slot(*decision) == slot) + { + continue; + } + let Ok(decision) = self.projection.direct_decision(slot) else { + continue; + }; + if matches!(decision, ProjectionDecisionV1::Undecided { .. }) { + continue; + } + if self.projected_decisions.insert(decision) { + self.pending_projection_decisions.push(decision); + } + } + } + fn ensure_live(&self) -> Result<(), ShadowErrorV1> { if self.poisoned || self.wal.is_poisoned() { Err(ShadowErrorV1::Poisoned) @@ -1161,10 +1420,12 @@ impl StarfishRbcDagShadowV1 { input: &ShadowInputV1, batch_sequence: u64, ) -> Result<(), ShadowErrorV1> { - let reference = input.candidate().reference(); - let slot = carrier_slot(reference); + let reference = input.candidate().map(CandidateCarrierV1::reference); + let slot = reference.map(carrier_slot); let violation = match input { ShadowInputV1::AuthenticatedIngress { .. } => { + let reference = reference.expect("authenticated ingress has a candidate"); + let slot = slot.expect("authenticated ingress has a slot"); if round_is_stale(self.model.local_carrier_round(), reference.round) { Some("stale authenticated ingress") } else if self.authenticated_slots.contains_key(&slot) { @@ -1174,6 +1435,8 @@ impl StarfishRbcDagShadowV1 { } } ShadowInputV1::CandidateRetention(_) => { + let reference = reference.expect("candidate retention has a candidate"); + let slot = slot.expect("candidate retention has a slot"); if round_is_stale(self.model.local_carrier_round(), reference.round) { Some("stale candidate retention") } else if self.candidates.contains_key(&reference) @@ -1186,6 +1449,8 @@ impl StarfishRbcDagShadowV1 { } } ShadowInputV1::CandidateRecovery(_) => { + let reference = reference.expect("candidate recovery has a candidate"); + let slot = slot.expect("candidate recovery has a slot"); let limit = self .committee .committee() @@ -1207,6 +1472,9 @@ impl StarfishRbcDagShadowV1 { } } ShadowInputV1::LocalOutbound(_) => None, + ShadowInputV1::DataAvailable(reference) => { + (!self.candidates.contains_key(reference)).then_some("unknown available carrier") + } }; if let Some(reason) = violation { return Err(ShadowErrorV1::ReplayPolicyViolation { @@ -1218,26 +1486,41 @@ impl StarfishRbcDagShadowV1 { } fn record_committed_input(&mut self, input: &ShadowInputV1, effects: &[ModelEffect]) { - let candidate = input.candidate().clone(); - let reference = candidate.reference(); - let slot = carrier_slot(reference); - self.candidates.insert(reference, candidate); - self.slot_candidates - .entry(slot) - .or_default() - .insert(reference); - match input { - ShadowInputV1::AuthenticatedIngress { .. } | ShadowInputV1::LocalOutbound(_) => { - self.authenticated_slots.entry(slot).or_insert(reference); + if let Some(candidate) = input.candidate().cloned() { + let reference = candidate.reference(); + let slot = carrier_slot(reference); + self.projection + .stage_carrier(candidate.clone()) + .expect("durably validated carrier must match projection committee"); + if candidate.header().application_header().is_none() || input.is_local() { + self.projection + .mark_data_available(reference) + .expect("staged control carrier is available"); } - ShadowInputV1::CandidateRetention(_) => { - self.ordinarily_retained_slots - .entry(slot) - .or_insert(reference); + self.candidates.insert(reference, candidate); + self.slot_candidates + .entry(slot) + .or_default() + .insert(reference); + match input { + ShadowInputV1::AuthenticatedIngress { .. } | ShadowInputV1::LocalOutbound(_) => { + self.authenticated_slots.entry(slot).or_insert(reference); + } + ShadowInputV1::CandidateRetention(_) => { + self.ordinarily_retained_slots + .entry(slot) + .or_insert(reference); + } + ShadowInputV1::CandidateRecovery(_) => {} + ShadowInputV1::DataAvailable(_) => unreachable!("handled without a candidate"), } - ShadowInputV1::CandidateRecovery(_) => {} + self.requested_recoveries.remove(&reference); + } + if let ShadowInputV1::DataAvailable(reference) = input { + self.projection + .mark_data_available(*reference) + .expect("model accepted availability only for a staged carrier"); } - self.requested_recoveries.remove(&reference); for effect in effects { match effect { ModelEffect::NeedCarrier { target, holders } => { @@ -1246,10 +1529,14 @@ impl StarfishRbcDagShadowV1 { ModelEffect::Delivered(delivered) => { self.delivered.insert(*delivered); self.requested_recoveries.remove(delivered); + self.projection + .mark_delivered(*delivered) + .expect("model delivery must name a staged carrier"); } ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} } } + self.drive_certified_projection(); } fn decode_batch(&self, records: &[Vec]) -> Result { @@ -1295,6 +1582,13 @@ impl StarfishRbcDagShadowV1 { Ok(ShadowInputV1::CandidateRecovery(candidate)) } } + RECORD_DATA_AVAILABLE => { + ensure_trace_tail(&decoded[1..])?; + let mut payload = RawDecoder::new(&decoded[0].payload); + let reference = payload.read_reference()?; + payload.finish()?; + Ok(ShadowInputV1::DataAvailable(reference)) + } RECORD_LOCAL_OUTBOUND_CONTENT => { if decoded.len() != 4 { return Err(ShadowErrorV1::InvalidBatch( @@ -1473,9 +1767,15 @@ fn validate_journal_transition( candidate: authenticated.candidate().clone(), }); } + ShadowInputV1::DataAvailable(_) => {} } - let local_reference = input.is_local().then(|| input.candidate().reference()); + let local_reference = input.is_local().then(|| { + input + .candidate() + .expect("local input has candidate") + .reference() + }); for entry in trace { let event = match entry { ModelTraceEvent::AdmissionLocked(target) if Some(*target) == local_reference => { @@ -1530,6 +1830,22 @@ fn validate_journal_transition( context, reference: *reference, }), + ModelTraceEvent::ConsensusSlotLocked { + consensus_round, + enclosing_carrier, + } => Some(JournalEventV1::LockConsensusSlot { + context, + consensus_round: *consensus_round, + enclosing_carrier: *enclosing_carrier, + }), + ModelTraceEvent::LeaderChoiceLocked { + consensus_round, + choice, + } => Some(JournalEventV1::LockLeaderChoice { + context, + consensus_round: *consensus_round, + choice: *choice, + }), ModelTraceEvent::DeliveryLocked(target) => Some(JournalEventV1::LockDelivery { context, target: *target, @@ -1607,6 +1923,16 @@ fn encode_batch( &authenticated.candidate().canonical_wire_bytes()?, )?); } + ShadowInputV1::DataAvailable(reference) => { + let mut payload = Vec::new(); + push_reference(&mut payload, *reference); + records.push(encode_raw_record( + context, + own_authority, + RECORD_DATA_AVAILABLE, + &payload, + )?); + } } records.push(encode_raw_record( context, @@ -1823,6 +2149,22 @@ fn encode_trace(trace: &ModelTraceEvent) -> Result, ShadowCodecErrorV1> bytes.push(TRACE_LOCAL_CARRIER_FIXED); push_reference(&mut bytes, *reference); } + ModelTraceEvent::ConsensusSlotLocked { + consensus_round, + enclosing_carrier, + } => { + bytes.push(TRACE_CONSENSUS_SLOT_LOCKED); + bytes.extend_from_slice(&consensus_round.to_be_bytes()); + push_reference(&mut bytes, *enclosing_carrier); + } + ModelTraceEvent::LeaderChoiceLocked { + consensus_round, + choice, + } => { + bytes.push(TRACE_LEADER_CHOICE_LOCKED); + bytes.extend_from_slice(&consensus_round.to_be_bytes()); + push_leader_choice(&mut bytes, *choice); + } ModelTraceEvent::DeliveryLocked(reference) => { bytes.push(TRACE_DELIVERY_LOCKED); push_reference(&mut bytes, *reference); @@ -1855,6 +2197,14 @@ fn decode_trace( next_index: decoder.read_u32()? as usize, }, TRACE_LOCAL_CARRIER_FIXED => ModelTraceEvent::LocalCarrierFixed(decoder.read_reference()?), + TRACE_CONSENSUS_SLOT_LOCKED => ModelTraceEvent::ConsensusSlotLocked { + consensus_round: decoder.read_u32()?, + enclosing_carrier: decoder.read_reference()?, + }, + TRACE_LEADER_CHOICE_LOCKED => ModelTraceEvent::LeaderChoiceLocked { + consensus_round: decoder.read_u32()?, + choice: decoder.read_leader_choice()?, + }, TRACE_DELIVERY_LOCKED => ModelTraceEvent::DeliveryLocked(decoder.read_reference()?), TRACE_EFFECT => ModelTraceEvent::Effect(decoder.read_effect(committee_size)?), other => return Err(ShadowCodecErrorV1::InvalidTrace(other)), @@ -1905,6 +2255,28 @@ fn push_phase(bytes: &mut Vec, statement: RbcPhaseStatementV1) { } } +fn push_consensus_reference(bytes: &mut Vec, reference: ConsensusVertexReference) { + push_reference(bytes, reference.carrier()); + bytes.extend_from_slice(&reference.consensus_round().to_be_bytes()); +} + +fn push_leader_choice(bytes: &mut Vec, choice: LeaderChoiceV1) { + match choice { + LeaderChoiceV1::Vote { leader } => { + bytes.push(0); + push_consensus_reference(bytes, leader); + } + LeaderChoiceV1::NoVote { + leader_author, + leader_round, + } => { + bytes.push(1); + bytes.extend_from_slice(&leader_author.to_be_bytes()); + bytes.extend_from_slice(&leader_round.to_be_bytes()); + } + } +} + fn encode_provenance(bytes: &mut Vec, provenance: IngressProvenanceV1) { match provenance { IngressProvenanceV1::DirectFromAuthor => bytes.push(PROVENANCE_DIRECT), @@ -1985,6 +2357,30 @@ fn ambiguous_slots( .collect() } +fn projection_error_is_pending(error: &CertifiedProjectionError) -> bool { + matches!( + error, + CertifiedProjectionError::CarrierNotDelivered(_) + | CertifiedProjectionError::CarrierDataUnavailable(_) + | CertifiedProjectionError::CarrierOutsideClosedPrefix(_) + | CertifiedProjectionError::MissingStrongParent(_) + | CertifiedProjectionError::FrontierNotClosed { .. } + ) +} + +fn projection_decision_slot(decision: ProjectionDecisionV1) -> LeaderSlotV1 { + match decision { + ProjectionDecisionV1::DirectCommit { leader } + | ProjectionDecisionV1::IndirectCommit { leader, .. } => LeaderSlotV1 { + author: leader.author(), + round: leader.consensus_round(), + }, + ProjectionDecisionV1::DirectSkip { slot } + | ProjectionDecisionV1::IndirectSkip { slot, .. } + | ProjectionDecisionV1::Undecided { slot } => slot, + } +} + struct RawDecoder<'a> { bytes: &'a [u8], position: usize, @@ -2050,6 +2446,25 @@ impl<'a> RawDecoder<'a> { } } + fn read_consensus_reference(&mut self) -> Result { + let carrier = self.read_reference()?; + let consensus_round = self.read_u32()?; + Ok(ConsensusVertexReference::new(carrier, consensus_round)) + } + + fn read_leader_choice(&mut self) -> Result { + match self.read_u8()? { + 0 => Ok(LeaderChoiceV1::Vote { + leader: self.read_consensus_reference()?, + }), + 1 => Ok(LeaderChoiceV1::NoVote { + leader_author: self.read_u16()?, + leader_round: self.read_u32()?, + }), + other => Err(ShadowCodecErrorV1::InvalidLeaderChoice(other)), + } + } + fn read_effect(&mut self, committee_size: usize) -> Result { match self.read_u8()? { EFFECT_NEED_CARRIER => { @@ -2255,7 +2670,7 @@ mod tests { assert_eq!(node.local_outbound_envelope(1), None); let before = node.wal_counts(); - let (heartbeat, effects) = node.create_local_control_heartbeat(123).unwrap(); + let (heartbeat, effects) = node.create_local_control_heartbeat(123, true).unwrap(); assert!(effects.is_empty()); assert_eq!(node.wal_counts().0, before.0 + 1); let candidate = decode_candidate( @@ -2273,7 +2688,19 @@ mod tests { assert_eq!(candidate.header().creation_time_ns(), 123); assert!(candidate.header().data_acknowledgments().is_empty()); assert!(candidate.header().phase_batch().is_empty()); - assert!(candidate.header().consensus_vertex().is_none()); + let vertex = candidate + .header() + .consensus_vertex() + .expect("first autonomous heartbeat carries the genesis projection"); + assert_eq!(vertex.consensus_round(), 1); + assert_eq!( + node.journal.snapshot().consensus_slot(1), + Some(heartbeat.reference()) + ); + assert_eq!( + node.journal.snapshot().leader_choice(1), + Some(vertex.leader_choice()) + ); assert_eq!(node.local_outbound_envelope(1), Some(heartbeat.clone())); assert_eq!(node.local_outbound_envelope(2), None); assert_eq!( @@ -2289,7 +2716,7 @@ mod tests { let durable_counts = node.wal_counts(); assert!(matches!( - node.create_local_control_heartbeat(124), + node.create_local_control_heartbeat(124, true), Err(ShadowErrorV1::Model(ModelError::LocalCarrierAlreadyFixed( 1 ))) @@ -2305,7 +2732,7 @@ mod tests { fn autonomous_control_heartbeat_advances_sequentially_and_reopens_exact_bytes() { let mut network = TestNetwork::new(); let first = network.nodes[0] - .create_local_control_heartbeat(1_000) + .create_local_control_heartbeat(1_000, true) .unwrap() .0; for author in [1, 2] { @@ -2334,7 +2761,7 @@ mod tests { assert_eq!(network.nodes[0].current_round_admitted_author_count(), 0); let second = network.nodes[0] - .create_local_control_heartbeat(2_000) + .create_local_control_heartbeat(2_000, true) .unwrap() .0; let second_candidate = decode_candidate( @@ -2374,6 +2801,15 @@ mod tests { assert!(!restarted.can_create_carrier()); assert_eq!(restarted.local_outbound_envelope(1), Some(first)); assert_eq!(restarted.local_outbound_envelope(2), Some(second)); + let first_reference = restarted + .local_outbound_envelope(1) + .expect("round one survived restart") + .reference(); + assert_eq!( + restarted.journal.snapshot().consensus_slot(1), + Some(first_reference) + ); + assert!(restarted.journal.snapshot().leader_choice(1).is_some()); } #[test] diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 0f3b784d..acd0fbeb 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! Async, non-authoritative network adapter for the persisted RBC-DAG shadow. +//! Async network adapter for the persisted RBC-DAG research runtime. use std::{ collections::{BTreeMap, BTreeSet}, @@ -33,8 +33,10 @@ use crate::{ }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ - MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, + ConsensusVertexReference, MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, + RbcDagContextV1, model::{ModelEffect, ModelError}, + projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ @@ -68,9 +70,10 @@ const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_milli /// Runtime role of the persisted carrier actor. /// /// Mirror mode preserves milestone three's one-to-one comparison against -/// direct Starfish-RBC headers. Autonomous mode opens an independent, -/// heartbeat-only carrier clock. It remains observational: neither mode can -/// call the core dispatcher or mutate authoritative consensus state. +/// direct Starfish-RBC headers. Autonomous mode opens an independent carrier +/// clock and owns embedded-RBC certification plus clean projection decisions. +/// It deliberately does not call the legacy core dispatcher: replacing the +/// temporary application-output scaffold is the M7 boundary. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ShadowServiceModeV1 { DirectMirror, @@ -144,6 +147,7 @@ enum ShadowServiceMessageV1 { TopologyChanged, RetryRecovery, HeartbeatTick, + DataAvailabilityChanged, Shutdown(oneshot::Sender>), } @@ -157,6 +161,7 @@ pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { mode: ShadowServiceModeV1, desired_topology: Arc>>, desired_direct_deliveries: Arc>>, + desired_available_applications: Arc>>, invalidated_by_overload: Arc>>, } @@ -280,6 +285,34 @@ impl StarfishRbcDagShadowServiceHandleV1 { } } + pub(crate) fn application_data_available( + &self, + application: BlockReference, + ) -> Result<(), ShadowServiceErrorV1> { + if !self.mode.is_autonomous() { + return Ok(()); + } + if application.authority as usize >= self.committee_size { + return Err(ShadowServiceErrorV1::UnknownAuthority( + application.authority, + )); + } + if !self + .desired_available_applications + .lock() + .insert(application) + { + return Ok(()); + } + match self + .sender + .try_send(ShadowServiceMessageV1::DataAvailabilityChanged) + { + Ok(()) | Err(TrySendError::Full(_)) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), + } + } + pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { self.update_peer(peer, true) } @@ -341,6 +374,7 @@ impl ShadowServiceMessageV1 { Self::TopologyChanged => "topology_changed", Self::RetryRecovery => "recovery_retry", Self::HeartbeatTick => "heartbeat_tick", + Self::DataAvailabilityChanged => "data_availability_changed", Self::Shutdown(_) => "shutdown", } } @@ -372,6 +406,8 @@ pub(crate) enum ShadowServiceEventV1 { carrier: BlockReference, header: RbcCanonicalHeader, }, + VertexProjected(ConsensusVertexReference), + LeaderDecided(ProjectionDecisionV1), Comparison(ShadowDeliveryComparisonV1), Input { kind: &'static str, @@ -661,6 +697,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let (event_tx, event_rx) = mpsc::channel(SHADOW_SERVICE_EVENT_CAPACITY_V1); let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); + let desired_available_applications = Arc::new(Mutex::new(BTreeSet::new())); let invalidated_by_overload = Arc::new(Mutex::new(None)); let retry_notification_pending = Arc::new(AtomicBool::new(false)); let heartbeat_notification_pending = Arc::new(AtomicBool::new(false)); @@ -722,6 +759,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let startup_events = event_tx.clone(); let actor_desired_topology = Arc::clone(&desired_topology); let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); + let actor_desired_available_applications = Arc::clone(&desired_available_applications); let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); let actor_retry_notification_pending = Arc::clone(&retry_notification_pending); let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); @@ -940,10 +978,12 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( connected: BTreeSet::new(), desired_topology: actor_desired_topology, desired_direct_deliveries: actor_desired_direct_deliveries, + desired_available_applications: actor_desired_available_applications, observed_topology: BTreeMap::new(), invalidated_by_overload: actor_invalidated_by_overload, pending_local, assigned_applications, + available_applications: BTreeSet::new(), pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), @@ -987,6 +1027,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( mode, desired_topology, desired_direct_deliveries, + desired_available_applications, invalidated_by_overload, }, event_rx, @@ -1068,10 +1109,12 @@ struct ShadowServiceStateV1 { connected: BTreeSet, desired_topology: Arc>>, desired_direct_deliveries: Arc>>, + desired_available_applications: Arc>>, observed_topology: BTreeMap, invalidated_by_overload: Arc>>, pending_local: BTreeMap, assigned_applications: BTreeSet, + available_applications: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, @@ -1289,12 +1332,59 @@ impl ShadowServiceStateV1 { self.emit(ShadowServiceEventV1::PendingRecovery( self.pending_recovery.len(), )); + self.reconcile_data_availability(); self.report_new_shadow_deliveries(); + self.report_projection_progress(); self.flush_carrier_sync_requests(false); self.emit_clock_state(); } - fn try_create_autonomous_carrier(&mut self) { + fn reconcile_data_availability(&mut self) { + if !self.mode.is_autonomous() { + return; + } + let mut desired = self.desired_available_applications.lock().clone(); + desired.extend(self.core.intrinsically_available_applications()); + for application in desired { + let carriers = self.core.application_carriers(application); + if carriers.is_empty() { + continue; + } + for carrier in &carriers { + if self.core.carrier_data_available(*carrier) { + continue; + } + let before = self.core.wal_counts(); + match self.core.mark_carrier_data_available(*carrier) { + Ok(effects) => { + self.report_wal_delta(before); + self.process_effects(effects); + } + Err(error) => { + self.mark_fatal(error); + return; + } + } + } + if carriers + .iter() + .all(|carrier| self.core.carrier_data_available(*carrier)) + { + self.available_applications.insert(application); + } + } + } + + fn report_projection_progress(&mut self) { + for projected in self.core.drain_projected_vertices() { + self.emit(ShadowServiceEventV1::VertexProjected(projected)); + } + for decision in self.core.drain_projection_decisions() { + self.emit(ShadowServiceEventV1::LeaderDecided(decision)); + } + } + + fn try_create_autonomous_carrier(&mut self, allow_no_vote: bool) { if !self.mode.is_autonomous() || !self.core.can_create_carrier() { self.emit_clock_state(); return; @@ -1312,8 +1402,11 @@ impl ShadowServiceStateV1 { Some(application) => self.core.create_local_application_carrier( application.application_header.clone(), creation_time_ns, + allow_no_vote, ), - None => self.core.create_local_control_heartbeat(creation_time_ns), + None => self + .core + .create_local_control_heartbeat(creation_time_ns, allow_no_vote), }; match result { Ok((envelope, effects)) => { @@ -1363,7 +1456,7 @@ impl ShadowServiceStateV1 { fn drive_autonomous_catch_up(&mut self) { while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(); + self.try_create_autonomous_carrier(true); if self.core.local_carrier_round() == round_before { break; } @@ -1466,7 +1559,7 @@ impl ShadowServiceStateV1 { && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(); + self.try_create_autonomous_carrier(false); if self.core.local_carrier_round() == round_before { break; } @@ -2078,7 +2171,10 @@ fn run_shadow_service( state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); } - ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(), + ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(true), + ShadowServiceMessageV1::DataAvailabilityChanged => { + state.reconcile_data_availability(); + } ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); @@ -2369,6 +2465,8 @@ mod tests { deliveries: &mut [usize], application_deliveries: &mut [BTreeSet], sync_requests: &mut usize, + projected_vertices: &mut usize, + projected_decisions: &mut usize, target_open_round: RoundNumber, ) { timeout(EVENT_TIMEOUT, async { @@ -2427,6 +2525,12 @@ mod tests { } => { application_deliveries[sender].insert(header.reference()); } + ShadowServiceEventV1::VertexProjected(_) => { + *projected_vertices = projected_vertices.saturating_add(1); + } + ShadowServiceEventV1::LeaderDecided(_) => { + *projected_decisions = projected_decisions.saturating_add(1); + } ShadowServiceEventV1::Rejected { error, .. } if error.contains("FutureCarrierOutsideBuffer") || error.contains("unexpected shadow response") => {} @@ -2686,6 +2790,8 @@ mod tests { let mut deliveries = vec![0; n]; let mut application_deliveries = vec![BTreeSet::new(); n]; let mut sync_requests = 0; + let mut projected_vertices = 0; + let mut projected_decisions = 0; for (authority, handle) in handles.iter().enumerate() { for peer in 0..n { if peer != authority { @@ -2693,7 +2799,7 @@ mod tests { } } } - for fixed_round in 1..=6 { + for fixed_round in 1..=18 { for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -2704,6 +2810,8 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, fixed_round + 1, ) .await; @@ -2713,7 +2821,12 @@ mod tests { deliveries.iter().all(|count| *count > 0), "every node must RBC-deliver mature heartbeat carriers: {deliveries:?}" ); - assert!(open_rounds.iter().all(|round| *round >= 7)); + assert!(open_rounds.iter().all(|round| *round >= 19)); + assert!(projected_vertices >= n * 3); + assert!( + projected_decisions > 0, + "clean projection did not decide: vertices={projected_vertices}, rounds={open_rounds:?}" + ); assert_eq!( sync_requests, 0, "healthy proactive rounds must not trigger repair polling" @@ -2775,8 +2888,11 @@ mod tests { } } + // Empty application commitments have no shard reconstruction event. + // They must become intrinsically data-available from their exact + // canonical header or the certified projection stalls behind them. let applications = (0..N as AuthorityIndex) - .map(|authority| direct_header(authority, 1, 0x70 + authority as u8)) + .map(|authority| direct_header(authority, 1, 0)) .collect::>(); let expected = applications .iter() @@ -2790,6 +2906,8 @@ mod tests { let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; let mut sync_requests = 0; + let mut projected_vertices = 0; + let mut projected_decisions = 0; pump_autonomous_until_round( &handles, &mut events, @@ -2797,6 +2915,8 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, 5, ) .await; @@ -2812,6 +2932,10 @@ mod tests { "application-critical phase carriers must not wait for a heartbeat tick" ); assert_eq!(sync_requests, 0); + assert!( + projected_vertices >= N, + "empty embedded applications must not stall clean projection" + ); drop(events); for handle in &handles { @@ -2965,6 +3089,8 @@ mod tests { let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; let mut sync_requests = 0; + let mut projected_vertices = 0; + let mut projected_decisions = 0; for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -2975,6 +3101,8 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, 2, ) .await; @@ -3026,6 +3154,8 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, 10, ) .await; @@ -3618,6 +3748,7 @@ mod tests { input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), }; for round in 0..input_capacity { @@ -3668,6 +3799,7 @@ mod tests { input_capacity: 1, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), }; assert!(matches!( @@ -3702,6 +3834,7 @@ mod tests { input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::clone(&invalidated), }; for peer in 1..LARGE_N { diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 84660acf..59e8de2c 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -366,7 +366,14 @@ impl Syncer { .with_label_values(&["local", "dropped"]) .inc(); tracing::warn!( - "Failed to enqueue non-authoritative RBC-DAG shadow carrier; comparison is invalid: {error}" + "Failed to enqueue RBC-DAG application carrier; the research run is invalid: {error}" + ); + } else if let Err(error) = + shadow.application_data_available(canonical.reference()) + { + self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + tracing::warn!( + "Failed to record local RBC-DAG application availability: {error}" ); } } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index c53364e6..818b9fba 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -586,7 +586,7 @@ mod smoke_tests { } if autonomous_clock { - tokio::time::timeout(timeout, async { + let autonomous_progress = tokio::time::timeout(timeout, async { loop { if validators.iter().all(|validator| { let metrics = validator.metrics(); @@ -606,6 +606,12 @@ mod smoke_tests { .with_label_values(&["delivery", "shadow"]) .get() > 0 + && metrics.starfish_rbc_dag_projected_vertices_total.get() > 0 + && metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get() + > 0 && (!embedded_rbc_authority || metrics .starfish_rbc_dag_shadow_inputs_total @@ -619,13 +625,46 @@ mod smoke_tests { time::sleep(Duration::from_millis(25)).await; } }) - .await - .expect("autonomous carrier clock did not advance while direct RBC committed"); + .await; + if autonomous_progress.is_err() { + let state = validators + .iter() + .map(|validator| { + let metrics = validator.metrics(); + ( + metrics.starfish_rbc_dag_shadow_clock_valid.get(), + metrics.starfish_rbc_dag_shadow_carrier_round.get(), + metrics.starfish_rbc_dag_projected_vertices_total.get(), + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get(), + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get(), + metrics.starfish_rbc_dag_shadow_pending_recovery.get(), + ) + }) + .collect::>(); + panic!( + "autonomous carrier projection did not advance while direct RBC committed; \ + per-node (valid, round, projected, commits, deliveries, recovery)={state:?}" + ); + } for validator in &validators { let metrics = validator.metrics(); assert_eq!(metrics.starfish_rbc_dag_shadow_clock_valid.get(), 1); assert!(metrics.starfish_rbc_dag_shadow_carrier_round.get() > 3); + assert!(metrics.starfish_rbc_dag_projected_vertices_total.get() > 0); + assert!( + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get() + > 0 + ); assert_eq!( metrics.starfish_rbc_dag_shadow_comparison_valid.get(), 0, diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index c593b25c..15d5f838 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -72,12 +72,12 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Run the persisted, non-authoritative RBC-DAG shadow alongside - /// `starfish-rbc`. + /// Enable the persisted RBC-DAG research runtime alongside + /// `starfish-rbc` (comparison-only unless autonomous mode is enabled). #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + /// Run the independent Starfish-RBC-DAG carrier clock and certified + /// projection. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, /// Let embedded carrier ECHO/READY delivery certify application @@ -117,12 +117,12 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Run the persisted, non-authoritative RBC-DAG shadow alongside - /// `starfish-rbc`. + /// Enable the persisted RBC-DAG research runtime alongside + /// `starfish-rbc` (comparison-only unless autonomous mode is enabled). #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + /// Run the independent Starfish-RBC-DAG carrier clock and certified + /// projection. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, /// Let embedded carrier ECHO/READY delivery certify application @@ -184,12 +184,12 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Run the persisted, non-authoritative RBC-DAG shadow alongside - /// `starfish-rbc`. + /// Enable the persisted RBC-DAG research runtime alongside + /// `starfish-rbc` (comparison-only unless autonomous mode is enabled). #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. + /// Run the independent Starfish-RBC-DAG carrier clock and certified + /// projection. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, /// Let embedded carrier ECHO/READY delivery certify application diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 764467bd..b2106bc3 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,7 +1,7 @@ # Starfish-RBC-DAG protocol design -Status: milestone-five authoritative embedded-RBC prototype; consensus projection and end-to-end -safety/liveness proof remain incomplete +Status: milestone-six certified-projection prototype; frontier output and end-to-end safety/liveness +proof remain incomplete The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not implemented. The staged prototype runs under `starfish-rbc`: direct-header comparison uses @@ -10,9 +10,11 @@ implemented. The staged prototype runs under `starfish-rbc`: direct-header compa only application-header certification authority with `--starfish-rbc-dag-embedded-rbc-authority`. Direct INIT still transports the application payload, but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add -`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Consensus -projection, commit, and output still use the existing Starfish DAG. The eventual protocol is new, -not a transport option or a version-two alias for `starfish-rbc`. +`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Autonomous +carriers now create durably locked logical consensus vertices and the clean projection produces +Starfish commit/skip decisions. Deterministic frontier output still uses the existing Starfish DAG +until milestone seven. The eventual protocol is new, not a transport option or a version-two alias +for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -52,13 +54,15 @@ containing the exact canonical application header, durable application-origin re immediate application/phase scheduling, and an opt-in authority boundary that prevents direct ECHO/READY from certifying a header. Idle control heartbeats use the same resolved leader timeout as the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application carriers and their -encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. +encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. Milestone six +adds independently numbered consensus vertices, quorum strong parents, objective Vote/NoVote +choices, exact contiguous delivery frontiers, durable local consensus locks, and a live committer +that consumes only RBC-delivered, data-available projected vertices. -The current authoritative mode changes only header certification. Consensus vertices, certified -projection, commits, and application ordering remain on the existing Starfish DAG. Direct INIT is -still the application-payload transport and is not a certification vote. Replacing that remaining -wrapper with a payload-only path and moving consensus into the clean carrier projection are later -milestones. +The current authoritative mode changes header certification and produces certified carrier-DAG +leader decisions. Direct INIT is still the application-payload transport and is not a certification +vote. The legacy Starfish DAG remains only as the temporary application-output scaffold; replacing +its commit/output path with deterministic committed frontier deltas is milestone seven. Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode requires an identical recovered direct-header history, control-only autonomous history reopens @@ -920,11 +924,12 @@ Starfish-RBC; application and encodable phase carriers are immediate. The harnes generator warmup, snapshots cumulative counters at the active boundary, and drains final latency samples. -| Profile | Verdict | TPS | p50 block | p50 E2E | Outbound | +| Profile | Verdict | TPS | Block latency | E2E latency | Outbound | |---|---:|---:|---:|---:|---:| | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | | Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | +| Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | | Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted @@ -939,9 +944,15 @@ rounds 458–459, and ended with zero pending recovery. Its direct ECHO and READ the composed four-validator test also asserts zero such outbound messages. Its higher latency is not evidence for a bad 250 ms timer—the independent timer was removed, and the same Starfish timeout was used. It exposes the transitional clean-predecessor gate: the existing direct Starfish -DAG still serializes proposal creation on embedded delivery. Milestone six must replace that gate -with the optimistic carrier clock plus certified consensus projection before the complete protocol -can be expected to recover Starfish pipelining. +DAG still serializes proposal creation on embedded delivery. + +The milestone-six run reached carrier rounds 356–359 with 35,506 carrier deliveries, 8,059 +application deliveries, 8,487 clean projected vertices, 830 clean direct commits, and zero pending +recovery. It validates the certified-projection structure, not final performance. The logical +committer currently runs beside the legacy clean-predecessor/output path, so this transitional run +pays for both and its 5.1/8.7-second latency is a red flag rather than a protocol target. Milestone +seven must make committed frontier deltas the sole ordering/output path before latency is compared +as the complete RBC-DAG protocol. ## 19. Contained implementation milestones @@ -968,8 +979,10 @@ Every milestone is committed separately. in version-two carriers, durably reconcile their origins, schedule application/phase carriers immediately, and remove direct ECHO/READY/delivery authority. Direct INIT remains payload transport; composed tests assert zero direct ECHO/READY traffic and positive embedded delivery. -6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit - leader choice, contiguous delivery frontiers, and strict clean-only committer consumers. +6. **Certified consensus projection (implemented):** add optional independently numbered consensus + vertices, quorum strong parents, explicit timeout-bound leader choices, contiguous exact + delivery frontiers, durable slot/choice locks, and a live clean-only direct committer. Malformed + optional vertices do not poison their enclosing carrier. 7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct prefixes and anchors, and add late-node and crash/restart tests. 8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, From 65dd57e38eee19da664087703a0d0474e2c79d76 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:36:55 +0200 Subject: [PATCH 35/62] Make RBC-DAG frontiers authoritative --- README.md | 35 +- crates/orchestrator/src/measurements.rs | 116 +++-- crates/starfish-core/src/block_handler.rs | 77 +++- crates/starfish-core/src/core.rs | 121 ++++- .../starfish-core/src/core_thread/spawned.rs | 30 ++ crates/starfish-core/src/metrics.rs | 77 +++- crates/starfish-core/src/net_sync.rs | 30 +- .../src/starfish_rbc_dag/journal.rs | 18 + .../src/starfish_rbc_dag/model.rs | 40 +- .../src/starfish_rbc_dag/projection.rs | 50 ++- .../src/starfish_rbc_dag_shadow.rs | 424 ++++++++++++------ .../src/starfish_rbc_dag_shadow_service.rs | 165 +++++-- crates/starfish-core/src/syncer.rs | 52 ++- crates/starfish-core/src/validator.rs | 59 ++- docs/starfish-rbc-dag-protocol.md | 55 ++- 15 files changed, 1059 insertions(+), 290 deletions(-) diff --git a/README.md b/README.md index ea0b261c..855c189c 100644 --- a/README.md +++ b/README.md @@ -57,14 +57,16 @@ the comparison shadow with `--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact application headers in version-two carriers and make embedded ECHO/READY/delivery their sole certification authority. Direct INIT remains payload transport, but direct ECHO/READY cannot clean -blocks in that mode. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for +blocks in that mode. Committed projected anchors now release deterministic carrier-frontier deltas, +and those deltas are the sole application ordering/output authority; the legacy Starfish committer +is disabled. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for Starfish-RBC by default); application and encodable phase carriers are emitted immediately. Autonomous carriers now embed durably locked consensus vertices with quorum strong parents, explicit Vote/NoVote choices, and exact delivery frontiers. Only RBC-delivered, data-available, -prefix-closed vertices enter the projection or its leader decisions. The existing Starfish DAG is -still the temporary application-output scaffold; the committed frontier linearizer is the next -milestone. Shadow traffic shares the +prefix-closed vertices enter the projection or its leader decisions. Frontier output retains exact +application references and is rebuilt from the ordered WAL on actor reopen. Full validator crash +recovery and proof-safe late-node state transfer remain outside this milestone. Shadow traffic shares the validator's network socket and bandwidth, and deployment requires a homogeneous new-binary committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; `--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean @@ -76,10 +78,10 @@ observational path was disabled or shed work and the comparison must be discarde production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between the cumulative direct and shadow delivery counters. Autonomous runs instead require -`starfish_rbc_dag_shadow_clock_valid == 1`, heartbeat/WAL progress, advancing carrier rounds, -in-window embedded-RBC delivery, projected-vertex and clean projected-commit progress, and bounded -clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 -in autonomous mode. +`starfish_rbc_dag_shadow_clock_valid == 1`, local-carrier/WAL progress, advancing carrier rounds, +in-window local carrier, embedded-RBC delivery, projected-vertex, clean projected-commit, and +committed-frontier application progress, plus bounded clock-state gauges. The current queue budget +supports at most 60 validators in mirror mode and 20 in autonomous mode. A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared @@ -91,21 +93,30 @@ nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Sta | Autonomous comparison, direct RBC authoritative | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | | Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | +| Frontier output authority (milestone seven) | VALID 10/10 | 950.25 | 2,020.9 ms | 2,082.6 ms | 0.70 MB/s | The milestone-five run produced 10,722 embedded application deliveries, reached carrier rounds 458–459, and ended with zero pending recovery. It also proves that the earlier 250 ms experimental heartbeat was not the latency cause: application and phase carriers are already event-driven, and using the shared 600 ms timeout did not restore the direct baseline. The remaining slowdown is an expected warning about the transitional architecture—the old direct DAG still serializes proposal -creation on embedded RBC cleanliness. Milestone six now lets the optimistic carrier clock advance -independently and feeds only certified vertices into the logical committer; milestone seven must -remove the remaining legacy output gate by committing deterministic frontier deltas. +creation on embedded RBC cleanliness. Milestone six lets the optimistic carrier clock advance +independently and feeds only certified vertices into the logical committer; its result motivated +milestone seven's removal of the remaining legacy output gate. The milestone-six run reached carrier rounds 356–359 with 35,506 carrier deliveries, 8,059 application deliveries, 8,487 projected vertices, 830 clean direct commits, and zero pending recovery. Its further latency increase is a structural red flag, not a projection-speed claim: certified decisions currently run alongside the old clean-predecessor/output path, so the benchmark -still pays for both. The next measurement is meaningful only after milestone seven removes that +still pays for both. Milestone seven's measurement below is the first one after removing that legacy gate. +Milestone seven disables the legacy committer in embedded-authority mode, advances application +production from the optimistic carrier clock, and releases exact application headers only through +committed frontier deltas. Its run reached carrier round 795 on every validator, delivered 79,035 +application carriers, released 77,870 applications through 1,880 committed frontiers, and ended +with zero pending recovery. This recovers 60.4% of milestone six's block-latency regression and +75.9% of its E2E regression, but 2.02/2.08 seconds is still well above the roughly 600 ms unsafe +Starfish-MAC target. The next performance work must measure and shorten the certified-projection +round/commit pipeline rather than reintroducing legacy certification or ordering. The local harness starts its timer after transaction-generator warmup, subtracts warmup counters, and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 600aab66..51af511c 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -695,6 +695,38 @@ impl MeasurementsCollection { .is_some_and(|(last, first)| last > first) } + fn count_bucket_sum_increased( + &self, + label: &str, + scraper_id: ScraperId, + buckets: &[&str], + ) -> bool { + let Some(series) = self.active_window_series(label, scraper_id) else { + return false; + }; + let bucket_is_monotonic = buckets.iter().all(|bucket| { + series.windows(2).all(|window| { + window[1].count_buckets.get(*bucket).copied().unwrap_or(0) + >= window[0].count_buckets.get(*bucket).copied().unwrap_or(0) + }) + }); + let totals = series + .iter() + .map(|measurement| { + buckets.iter().fold(0usize, |total, bucket| { + total.saturating_add( + measurement.count_buckets.get(*bucket).copied().unwrap_or(0), + ) + }) + }) + .collect::>(); + bucket_is_monotonic + && totals + .last() + .zip(totals.first()) + .is_some_and(|(last, first)| last > first) + } + fn count_bucket_is_always_zero( &self, label: &str, @@ -986,6 +1018,10 @@ impl MeasurementsCollection { .parameters .node_parameters .starfish_rbc_dag_autonomous_clock; + let embedded_rbc_authority = self + .parameters + .node_parameters + .starfish_rbc_dag_embedded_rbc_authority; let shadow_comparison_enabled = shadow_enabled && !shadow_autonomous_clock_enabled; let shadow_valid_scrapers = self .data @@ -1145,60 +1181,75 @@ impl MeasurementsCollection { .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); let autonomous_buffered_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let every_autonomous_scraper_has_progress = - shadow_autonomous_clock_valid_scrapers - .iter() - .all(|scraper_id| { - self.count_bucket_increased( + let every_autonomous_scraper_has_progress = shadow_autonomous_clock_valid_scrapers + .iter() + .all(|scraper_id| { + self.count_bucket_sum_increased( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + &["heartbeat,accepted", "application_carrier,accepted"], + ) && self.count_bucket_increased( + "starfish_rbc_dag_shadow_inputs_total", + *scraper_id, + "delivery,shadow", + ) && self.scalar_counter_increased( + "starfish_rbc_dag_shadow_wal_appended_batches_total", + *scraper_id, + ) && self.scalar_counter_increased( + "starfish_rbc_dag_shadow_wal_appended_records_total", + *scraper_id, + ) && self.scalar_counter_increased( + "starfish_rbc_dag_projected_vertices_total", + *scraper_id, + ) && self.count_bucket_increased( + "starfish_rbc_dag_projection_decisions_total", + *scraper_id, + "direct_commit", + ) && (!embedded_rbc_authority + || self.count_bucket_increased( "starfish_rbc_dag_shadow_inputs_total", *scraper_id, - "heartbeat,accepted", + "frontier,committed", ) && self.count_bucket_increased( "starfish_rbc_dag_shadow_inputs_total", *scraper_id, - "delivery,shadow", - ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_appended_batches_total", - *scraper_id, - ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_appended_records_total", - *scraper_id, - ) && self.scalar_counter_increased( - "starfish_rbc_dag_projected_vertices_total", - *scraper_id, - ) && self.count_bucket_increased( - "starfish_rbc_dag_projection_decisions_total", - *scraper_id, - "direct_commit", - ) && self.scalar_gauge_increased( + "frontier,application", + )) + && self.scalar_gauge_increased( "starfish_rbc_dag_shadow_carrier_round", *scraper_id, - ) && self.latest_scalar_greater_than( + ) + && self.latest_scalar_greater_than( "starfish_rbc_dag_shadow_carrier_round", *scraper_id, 1.0, - ) && self.latest_scalar_equals( + ) + && self.latest_scalar_equals( "starfish_rbc_dag_shadow_pending_recovery", *scraper_id, 0.0, - ) && self.gauge_always_at_most( + ) + && self.gauge_always_at_most( "starfish_rbc_dag_shadow_phase_backlog", *scraper_id, autonomous_phase_backlog_bound as f64, - ) && self.gauge_always_at_most( + ) + && self.gauge_always_at_most( "starfish_rbc_dag_shadow_admitted_authors", *scraper_id, self.parameters.nodes as f64, - ) && self.gauge_always_at_most( + ) + && self.gauge_always_at_most( "starfish_rbc_dag_shadow_admitted_stake", *scraper_id, f64::MAX, - ) && self.gauge_always_at_most( + ) + && self.gauge_always_at_most( "starfish_rbc_dag_shadow_buffered_authenticated", *scraper_id, autonomous_buffered_bound as f64, ) - }); + }); let shadow_autonomous_clock_valid = shadow_autonomous_clock_enabled && expected_shadow_nodes != 0 && shadow_autonomous_clock_valid_nodes == expected_shadow_nodes @@ -1601,10 +1652,15 @@ mod test { Measurement { timestamp, count_buckets: HashMap::from([ - ("heartbeat,accepted".to_owned(), wal_durable_records), + ( + "application_carrier,accepted".to_owned(), + wal_durable_records, + ), ("delivery,shadow".to_owned(), wal_durable_records), + ("frontier,committed".to_owned(), wal_durable_records), + ("frontier,application".to_owned(), wal_durable_records), ]), - count: wal_durable_records.saturating_mul(2), + count: wal_durable_records.saturating_mul(4), ..Measurement::default() }, ); diff --git a/crates/starfish-core/src/block_handler.rs b/crates/starfish-core/src/block_handler.rs index 001fa0a0..76bc4269 100644 --- a/crates/starfish-core/src/block_handler.rs +++ b/crates/starfish-core/src/block_handler.rs @@ -261,30 +261,23 @@ impl RealCommitHandler { let pending = committed.into_iter().skip(ready_count).collect(); (resulted_committed, pending) } -} -impl CommitObserver for RealCommitHandler { - fn handle_commit( + fn record_commit_metadata<'a>( &mut self, - dag_state: &DagState, - committed_leaders: Vec<(Data, Option)>, - ) -> Vec { - let mut committed = self - .commit_interpreter - .handle_commit(dag_state, committed_leaders); + committed: impl IntoIterator, + ) { let current_timestamp = runtime::timestamp_utc(); let metrics_active = self .metrics .metrics_active .load(std::sync::atomic::Ordering::Relaxed); - for commit in &committed { - self.committed_leaders.push(commit.0.anchor); + for commit in committed { + self.committed_leaders.push(commit.anchor); self.committed_count += 1; - // Chain rolling commit digest: hash(prev_digest || anchor.digest) let mut hasher = blake3::Hasher::new(); hasher.update(&self.commit_digest); - hasher.update(commit.0.anchor.digest.as_ref()); + hasher.update(commit.anchor.digest.as_ref()); self.commit_digest = *hasher.finalize().as_bytes(); let commit_index = self.committed_count; @@ -296,13 +289,12 @@ impl CommitObserver for RealCommitHandler { self.metrics.commit_digest.set(digest_short as i64); } - for block in &commit.0.blocks { - let gap = commit.0.anchor.round.saturating_sub(block.round()); + for block in &commit.blocks { + let gap = commit.anchor.round.saturating_sub(block.round()); self.metrics.commit_gap.observe(gap as f64); let block_creation_time = block.meta_creation_time(); let block_latency = current_timestamp.saturating_sub(block_creation_time); - if block_creation_time.is_zero() || block_latency.as_secs() > 60 { tracing::debug!( "Latency of block {} is too large, \ @@ -325,16 +317,9 @@ impl CommitObserver for RealCommitHandler { .committed_blocks .with_label_values(&[&block.authority().to_string()]) .inc(); - tracing::debug!("Latency of block {} is computed", block.reference()); } - // Tick the validator's `benchmark_duration` Prometheus counter, - // but only inside the active window. The orchestrator uses this - // counter as the timestamp on every scrape, which then becomes - // the denominator of the TPS rate. Anchor it to - // `active_start_micros` so warmup and wind-down seconds do not - // dilute the steady-state rate. if metrics_active { let active_start_micros = self .metrics @@ -349,6 +334,19 @@ impl CommitObserver for RealCommitHandler { } } } + } +} + +impl CommitObserver for RealCommitHandler { + fn handle_commit( + &mut self, + dag_state: &DagState, + committed_leaders: Vec<(Data, Option)>, + ) -> Vec { + let mut committed = self + .commit_interpreter + .handle_commit(dag_state, committed_leaders); + self.record_commit_metadata(committed.iter().map(|commit| &commit.0)); if dag_state.consensus_protocol == ConsensusProtocol::StarfishBls { let mut pending = dag_state.read_pending_not_certified(); pending.append(&mut committed); @@ -370,6 +368,39 @@ impl CommitObserver for RealCommitHandler { resulted_committed } + fn handle_rbc_dag_commit( + &mut self, + dag_state: &DagState, + anchor: BlockReference, + applications: &[BlockReference], + ) -> Vec { + let blocks = applications + .iter() + .map(|reference| { + let block = dag_state + .get_storage_block(*reference) + .unwrap_or_else(|| panic!("committed RBC-DAG application {reference} missing")); + assert!( + dag_state.is_data_available(reference), + "committed RBC-DAG application {reference} is unavailable" + ); + block + }) + .collect::>(); + let commit = CommittedSubDag::new(anchor, blocks); + self.record_commit_metadata(std::iter::once(&commit)); + for block in &commit.blocks { + if block.round() > 0 { + self.transaction_observer(block.clone()); + } + } + self.sequenced_commit_count += 1; + self.metrics + .commit_availability_gap + .set((self.committed_count - self.sequenced_commit_count) as i64); + vec![commit] + } + fn recover_committed( &mut self, committed: AHashSet, diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index 82dc5009..9d701ff3 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -72,6 +72,10 @@ pub struct Core { recovered_committed_leaders_count: Option, committer: UniversalCommitter, pub(crate) encoder: Encoder, + /// M7 application-production mode: direct Starfish headers are payload + /// descriptors only. Their dirty/clean DAG is no longer a consensus or + /// output authority, so raw threshold-clock progress may produce them. + rbc_dag_application_production: bool, } #[derive(Debug, Clone)] @@ -199,6 +203,7 @@ impl Core { recovered_committed_leaders_count: Some(committed_leaders_count), committer, encoder, + rbc_dag_application_production: false, }; if !unprocessed_blocks.is_empty() { @@ -489,7 +494,11 @@ impl Core { .utilization_timer .utilization_timer("Core::try_new_block"); - let proposal_round = self.dag_state.proposal_round(); + let proposal_round = if self.rbc_dag_application_production { + self.dag_state.threshold_clock_round() + } else { + self.dag_state.proposal_round() + }; tracing::debug!( "Attempt to construct block in round {} (proposal round {}). Current pending: {:?}", clock_round, @@ -506,7 +515,8 @@ impl Core { let protocol = self.dag_state.consensus_protocol; // Dual-DAG protocols: require clean parent quorum before creating a block. - if protocol.uses_dual_dag() + if !self.rbc_dag_application_production + && protocol.uses_dual_dag() && clock_round > 1 && !self.dag_state.clean_parent_quorum(clock_round - 1) { @@ -523,7 +533,8 @@ impl Core { // Starfish-RBC that local header is dirty until the local RBC instance // delivers it; another clean quorum must not let us smuggle this dirty // mandatory parent into a proposal. - if protocol.is_starfish_rbc() + if !self.rbc_dag_application_production + && protocol.is_starfish_rbc() && clock_round > 1 && self .last_own_block @@ -602,7 +613,8 @@ impl Core { clock_round ); } - if protocol.uses_dual_dag() + if !self.rbc_dag_application_production + && protocol.uses_dual_dag() && clock_round > 1 && block_references.is_empty() && !allows_minimal_refs @@ -764,7 +776,9 @@ impl Core { // otherwise a dirty child can suppress one of its clean parents and // then be filtered itself, shrinking the usable clean frontier. let (compression_candidates, deferred_dirty_refs): (Vec<_>, Vec<_>) = - if self.dag_state.consensus_protocol.is_starfish_rbc() { + if self.dag_state.consensus_protocol.is_starfish_rbc() + && !self.rbc_dag_application_production + { pending_refs.into_iter().partition(|reference| { reference.round == 0 || self.dag_state.has_clean_vertex(reference) }) @@ -779,7 +793,8 @@ impl Core { self.compress_pending_block_references(&compression_candidates, block_round); // Dual-DAG protocols: filter parents to only include clean blocks. - if self.dag_state.consensus_protocol.uses_dual_dag() { + if self.dag_state.consensus_protocol.uses_dual_dag() && !self.rbc_dag_application_production + { let before = block_references.clone(); block_references.retain(|r| r.round == 0 || self.dag_state.has_clean_vertex(r)); let filtered_out_refs: Vec<_> = before @@ -803,6 +818,7 @@ impl Core { let is_compressed_non_leader = self.dag_state.consensus_protocol.uses_compressed_refs() && self.committee.elect_leader(block_round) != self.authority; if self.dag_state.consensus_protocol.uses_dual_dag() + && !self.rbc_dag_application_production && block_round > 1 && !is_compressed_non_leader { @@ -1486,6 +1502,9 @@ impl Core { connected_authorities: &AHashSet, relaxed: bool, ) -> bool { + if self.rbc_dag_application_production { + return quorum_round > 0 && self.dag_state.threshold_clock_round() >= quorum_round; + } if quorum_round == 0 || self.dag_state.proposal_round() < quorum_round { return false; } @@ -1543,6 +1562,39 @@ impl Core { self.flush_pending_clean_refs(); } + /// Persist an M7 frontier delta without feeding the obsolete Starfish + /// clean-DAG commit/proposal watermarks back into block production. + pub fn handle_rbc_dag_committed_delta(&mut self, committed: Vec) { + let _timer = self + .metrics + .utilization_timer + .utilization_timer("Core::handle_rbc_dag_committed_delta"); + let mut commit_data = Vec::with_capacity(committed.len()); + for commit in &committed { + self.dag_state.update_last_committed_rounds(commit); + commit_data.push(CommitData::new( + commit, + self.dag_state.last_committed_rounds(), + )); + } + let store_start = std::time::Instant::now(); + self.store + .store_commits(commit_data) + .expect("Store RBC-DAG frontier commits should not fail"); + self.metrics + .store_commits_latency_us + .inc_by(store_start.elapsed().as_micros() as u64); + self.metrics.store_commits_count.inc(); + } + + pub(crate) fn enable_rbc_dag_application_production(&mut self) { + assert!( + self.dag_state.consensus_protocol.is_starfish_rbc(), + "RBC-DAG application production requires Starfish-RBC payload headers" + ); + self.rbc_dag_application_production = true; + } + pub fn write_commits(&mut self, _commits: &[CommitData]) {} pub fn take_recovered_committed(&mut self) -> (AHashSet, usize) { @@ -1861,6 +1913,63 @@ mod tests { })); } + #[test] + fn rbc_dag_application_production_does_not_wait_for_legacy_clean_delivery() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let private_config = NodePrivateConfig::new_for_tests(authority); + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + private_config, + metrics, + recovered, + None, + ); + core.enable_rbc_dag_application_production(); + + let own_round_one = core + .try_new_block("new_blocks") + .expect("round-one application header should be creatable"); + let peers = [1, 2] + .into_iter() + .map(|peer| make_starfish_rbc_round_1_block(&committee, peer)) + .collect::>(); + core.add_headers(peers, DataSource::BlockBundleStreamingHeader); + + assert_eq!(core.dag_state().threshold_clock_round(), 2); + assert!(!core.dag_state().has_clean_vertex(own_round_one.reference())); + let round_two = core + .try_new_block("new_blocks") + .expect("RBC-DAG application production must follow the raw threshold clock"); + assert_eq!(round_two.round(), 2); + assert!( + round_two + .block_references() + .contains(own_round_one.reference()) + ); + } + #[test] fn mysticeti_bls_non_leader_can_build_round_2_with_prev_leader_parent() { let authority = 0; diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 0b9a53f0..56f2fcad 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -14,6 +14,7 @@ use crate::{ data::Data, metrics::{Metrics, UtilizationTimerExt}, starfish_rbc::PinnedRbcHeader, + starfish_rbc_dag_shadow::CommittedFrontierDeltaV1, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ AuthorityIndex, BlockReference, ProvableShard, ReconstructedTransactionData, RoundNumber, @@ -71,6 +72,8 @@ enum CoreThreadCommand { ApplySailfishCertificates(Vec, oneshot::Sender<()>), /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), + /// Commit one deterministic clean carrier-frontier application delta. + ApplyStarfishRbcDagFrontier(CommittedFrontierDeltaV1, oneshot::Sender<()>), /// Store a Sailfish++ timeout certificate in DagState. ApplyTimeoutCert(SailfishTimeoutCert, oneshot::Sender<()>), /// Store a Sailfish++ no-vote certificate in DagState. @@ -211,6 +214,15 @@ impl CoreThread { self.syncer.apply_starfish_rbc_deliveries(delivered_headers); sender.send(()).ok(); } + CoreThreadCommand::ApplyStarfishRbcDagFrontier(delta, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_dag_frontier"]) + .inc(); + self.syncer.apply_starfish_rbc_dag_frontier(delta); + sender.send(()).ok(); + } CoreThreadCommand::ApplyTimeoutCert(cert, sender) => { metrics .core_thread_tasks_total @@ -466,6 +486,15 @@ mod tests { Vec::new() } + fn handle_rbc_dag_commit( + &mut self, + _dag_state: &DagState, + _anchor: BlockReference, + _applications: &[BlockReference], + ) -> Vec { + Vec::new() + } + fn recover_committed( &mut self, _committed: AHashSet, @@ -513,6 +542,7 @@ mod tests { None, None, None, + false, ); CoreThreadDispatcher::start(syncer) } diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index f99b0f4f..aabc76f7 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -267,9 +267,11 @@ pub struct MetricReporter { /// and immediately before the measured local-benchmark interval begins. #[derive(Clone, Copy, Debug, Default)] pub struct AutonomousClockBenchmarkBaseline { - accepted_heartbeats: u64, + accepted_local_carriers: u64, delivered_carriers: u64, delivered_applications: u64, + committed_frontiers: u64, + frontier_applications: u64, projected_vertices: u64, projection_decisions: u64, wal_batches: u64, @@ -297,6 +299,8 @@ struct AutonomousClockBenchmarkSummary { accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, + committed_frontiers: u64, + frontier_applications: u64, projected_vertices: u64, projection_decisions: u64, wal_batches: u64, @@ -341,7 +345,13 @@ fn summarize_autonomous_clock_benchmark( .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["heartbeat", "accepted"]) .get() - > baseline.accepted_heartbeats + .saturating_add( + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["application_carrier", "accepted"]) + .get(), + ) + > baseline.accepted_local_carriers && metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) @@ -352,7 +362,17 @@ fn summarize_autonomous_clock_benchmark( .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get() - > baseline.delivered_applications) + > baseline.delivered_applications + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get() + > baseline.committed_frontiers + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get() + > baseline.frontier_applications) && metrics.starfish_rbc_dag_projected_vertices_total.get() > baseline.projected_vertices && metrics @@ -416,6 +436,24 @@ fn summarize_autonomous_clock_benchmark( .get() }) .sum(); + let committed_frontiers = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get() + }) + .sum(); + let frontier_applications = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get() + }) + .sum(); let projected_vertices = metrics .iter() .map(|metrics| metrics.starfish_rbc_dag_projected_vertices_total.get()) @@ -496,6 +534,8 @@ fn summarize_autonomous_clock_benchmark( accepted_heartbeats, delivered_carriers, delivered_applications, + committed_frontiers, + frontier_applications, projected_vertices, projection_decisions, wal_batches, @@ -526,10 +566,15 @@ pub struct VecHistogramReporter { impl Metrics { pub fn autonomous_clock_benchmark_baseline(&self) -> AutonomousClockBenchmarkBaseline { AutonomousClockBenchmarkBaseline { - accepted_heartbeats: self + accepted_local_carriers: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["heartbeat", "accepted"]) - .get(), + .get() + .saturating_add( + self.starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["application_carrier", "accepted"]) + .get(), + ), delivered_carriers: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) @@ -538,6 +583,14 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get(), + committed_frontiers: self + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get(), + frontier_applications: self + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get(), projected_vertices: self.starfish_rbc_dag_projected_vertices_total.get(), projection_decisions: self .starfish_rbc_dag_projection_decisions_total @@ -1759,10 +1812,12 @@ impl Metrics { table.add_row(row![ b->"Clock/WAL progress:", format!( - "heartbeats={}, carrier deliveries={}, application deliveries={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", + "heartbeats={}, carrier deliveries={}, application deliveries={}, committed frontiers={}, frontier applications={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, summary.delivered_carriers, summary.delivered_applications, + summary.committed_frontiers, + summary.frontier_applications, summary.projected_vertices, summary.projection_decisions, summary.wal_batches, @@ -2447,7 +2502,7 @@ mod tests { let metrics = &metrics[0]; metrics .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) + .with_label_values(&["application_carrier", "accepted"]) .inc(); metrics .starfish_rbc_dag_shadow_inputs_total @@ -2480,6 +2535,14 @@ mod tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .inc(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .inc(); assert!( summarize_autonomous_clock_benchmark( &[Arc::clone(metrics)], diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index f1c37fa9..37e39241 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -1983,6 +1983,7 @@ impl NetworkSyncer } else { (None, None, None) }; + let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; let syncer = Syncer::new( core, NetworkSyncSignals { @@ -1995,6 +1996,7 @@ impl NetworkSyncer sf_msg_tx.clone(), starfish_rbc_service.clone(), starfish_rbc_dag_shadow_service.clone(), + embedded_rbc_authority, ); let initial_round = syncer.core().next_block_round(); let syncer = CoreThreadDispatcher::start(syncer); @@ -2212,7 +2214,6 @@ impl NetworkSyncer }) }); - let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; let embedded_rbc_committee_id = embedded_rbc_authority.then(|| { RbcCommitteeId::derive(&inner.committee) .expect("validated direct RBC committee must retain a stable identifier") @@ -2289,12 +2290,7 @@ impl NetworkSyncer embedded_rbc_committee_id .expect("embedded authority must cache its committee ID"), ) { - Ok(header) => { - event_inner - .syncer - .apply_starfish_rbc_deliveries(vec![header]) - .await; - } + Ok(_) => {} Err(error) => { shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); tracing::error!( @@ -2306,6 +2302,26 @@ impl NetworkSyncer } } } + ShadowServiceEventV1::FrontierCommitted(delta) => { + if embedded_rbc_authority { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .inc(); + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "carrier"]) + .inc_by(delta.carriers.len() as u64); + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .inc_by(delta.applications.len() as u64); + event_inner + .syncer + .apply_starfish_rbc_dag_frontier(delta) + .await; + } + } ShadowServiceEventV1::VertexProjected(reference) => { shadow_metrics .starfish_rbc_dag_projected_vertices_total diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index bdb4fb1d..a68f2fe7 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -1036,6 +1036,7 @@ pub struct WriteAheadJournalV1 { /// A transition checked against one exact journal prefix. Keeping only the /// newly appended events avoids cloning the complete durable history on every /// live shadow input. +#[cfg(test)] pub(crate) struct ValidatedJournalBatchV1 { base_event_count: usize, base_snapshot: Arc, @@ -1064,6 +1065,7 @@ impl WriteAheadJournalV1 { Ok(()) } + #[cfg(test)] pub(crate) fn validate_batch( &self, events: Vec, @@ -1080,6 +1082,7 @@ impl WriteAheadJournalV1 { }) } + #[cfg(test)] pub(crate) fn commit_validated_batch( &mut self, batch: ValidatedJournalBatchV1, @@ -1097,6 +1100,21 @@ impl WriteAheadJournalV1 { Ok(()) } + /// Apply a batch to the volatile authority snapshot without cloning its + /// complete retained history. This is safe only for a fail-stop caller + /// that persists the corresponding raw records before exposing any + /// effects and permanently poisons itself on a subsequent WAL failure. + pub(crate) fn apply_batch_unpublished( + &mut self, + events: Vec, + ) -> Result<(), JournalErrorV1> { + for event in events { + Arc::make_mut(&mut self.snapshot).apply(&event)?; + self.durable_events.push(event); + } + Ok(()) + } + pub fn record_authenticated_ingress( &mut self, authenticated: AuthenticatedCarrierV1, diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index d4f0e64f..344eccb1 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -19,7 +19,7 @@ use std::{ use crate::{ committee::Committee, - crypto::Blake3Hasher, + crypto::{Blake3Hasher, TransactionsCommitment}, types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, }; @@ -277,14 +277,14 @@ struct CarrierRecord { } impl CarrierRecord { - fn new(carrier: CandidateCarrierV1) -> Self { + fn new(carrier: CandidateCarrierV1, data_available: bool) -> Self { Self { carrier, authenticated: false, admitted: false, phase_batch_cursor: 0, delivered: false, - data_available: false, + data_available, prefix_closed: false, } } @@ -343,6 +343,7 @@ pub struct RbcDagModel { committee_id: RbcDagCommitteeId, context: RbcDagContextV1, own_authority: AuthorityIndex, + intrinsic_empty_data_available: bool, revision: u64, lineage: ModelLineage, local_carrier_round: RoundNumber, @@ -383,6 +384,7 @@ impl RbcDagModel { committee_id, context, own_authority, + intrinsic_empty_data_available: false, revision: 0, lineage: [0; 32], local_carrier_round: 1, @@ -399,6 +401,18 @@ impl RbcDagModel { }) } + /// Enable the runtime rule that a carrier with the canonical empty + /// transaction commitment needs no external reconstruction oracle. The + /// generic M2 model leaves this disabled so tests can control DA + /// independently of carrier contents. + pub fn enable_intrinsic_empty_data_availability(&mut self) { + assert!( + self.carriers.is_empty(), + "intrinsic availability mode must be fixed before ingress" + ); + self.intrinsic_empty_data_available = true; + } + pub fn own_authority(&self) -> AuthorityIndex { self.own_authority } @@ -461,6 +475,20 @@ impl RbcDagModel { self.apply_input_traced(input).map(|log| log.effects()) } + /// Apply a transition before any of its effects are externally exposed. + /// The durable actor uses this fail-stop path to avoid cloning the entire + /// retained reducer history for every carrier. If persistence of the + /// returned trace fails, the caller must poison and terminate the actor; + /// it must never publish the returned effects. + pub(crate) fn apply_input_unpublished( + &mut self, + input: ModelInputRecord, + ) -> Result<(Vec, Vec), ModelError> { + let log = self.apply_input_traced(input)?; + let effects = log.effects(); + Ok((log.trace, effects)) + } + /// Deterministically reconstruct a model by replaying the original typed /// inputs in their recorded order. No round or lock is synthesized. pub fn replay_from_records( @@ -780,9 +808,11 @@ impl RbcDagModel { log: &mut TransitionLog, ) { let reference = carrier.reference(); + let intrinsic_data_available = self.intrinsic_empty_data_available + && carrier.header().transactions_commitment() == TransactionsCommitment::default(); self.carriers .entry(reference) - .or_insert_with(|| CarrierRecord::new(carrier)); + .or_insert_with(|| CarrierRecord::new(carrier, intrinsic_data_available)); // Canonical content can satisfy a previously latched recovery even if // the receiver-specific authenticator is invalid. @@ -2534,7 +2564,7 @@ mod tests { let reference = carrier.reference(); model .carriers - .insert(reference, CarrierRecord::new(carrier)); + .insert(reference, CarrierRecord::new(carrier, false)); let mut candidate_state = RbcCandidateState::default(); candidate_state.readies.extend([1, 2]); let mut slot = RbcSlotState::default(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 21797741..5c17e5ee 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -140,6 +140,7 @@ pub struct CertifiedProjectionModel { delivered_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, closed_prefixes: Vec>, vertices: BTreeMap, + vertices_by_round: BTreeMap>, consensus_slots: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, committed_frontier: DeliveryFrontierV1, committed_anchors: BTreeSet, @@ -165,6 +166,7 @@ impl CertifiedProjectionModel { delivered_slots: BTreeMap::new(), closed_prefixes: vec![Vec::new(); committee_size], vertices: BTreeMap::new(), + vertices_by_round: BTreeMap::new(), consensus_slots: BTreeMap::new(), committed_frontier: vec![None; committee_size], committed_anchors: BTreeSet::new(), @@ -271,6 +273,24 @@ impl CertifiedProjectionModel { .map(|projected| &projected.vertex) } + pub fn is_committed_anchor(&self, reference: ConsensusVertexReference) -> bool { + self.committed_anchors.contains(&reference) + } + + /// Whether the current committed prefix already contains every carrier + /// named by `frontier`. This is used when a later anchor first resolves an + /// older slot: an intermediate leader can subsequently be decided as a + /// logical commit even though its entire payload frontier was already + /// output by that later anchor. + pub fn committed_frontier_dominates(&self, frontier: &[Option]) -> bool { + frontier.len() == self.committed_frontier.len() + && frontier + .iter() + .copied() + .zip(self.committed_frontier.iter().copied()) + .all(|(older, committed)| self.is_exact_extension(older, committed)) + } + /// Current exact closed carrier-prefix frontier in authority order. pub fn closed_frontier(&self) -> DeliveryFrontierV1 { self.committee @@ -392,6 +412,10 @@ impl CertifiedProjectionModel { effective_frontier, }, ); + self.vertices_by_round + .entry(vertex_reference.consensus_round()) + .or_default() + .insert(vertex_reference); self.consensus_slots .entry((author, vertex_reference.consensus_round())) .or_default() @@ -682,10 +706,15 @@ impl CertifiedProjectionModel { &self, round: RoundNumber, ) -> impl Iterator { - self.vertices - .iter() - .filter(move |(reference, _)| reference.consensus_round() == round) - .map(|(reference, projected)| (*reference, projected)) + self.vertices_by_round + .get(&round) + .into_iter() + .flatten() + .filter_map(|reference| { + self.vertices + .get(reference) + .map(|projected| (*reference, projected)) + }) } fn voter_authors( @@ -803,6 +832,10 @@ impl CertifiedProjectionModel { effective_frontier: vec![None; self.committee.len()], }, ); + self.vertices_by_round + .entry(reference.consensus_round()) + .or_default() + .insert(reference); self.consensus_slots .entry((reference.author(), reference.consensus_round())) .or_default() @@ -1088,6 +1121,11 @@ mod tests { let anchor_carrier = clean(&mut model, anchor); let anchor_vertex = model.try_project(anchor_carrier).unwrap(); model.record_committed_anchor(anchor_vertex).unwrap(); + assert!( + model.committed_frontier_dominates( + &carriers.iter().copied().map(Some).collect::>() + ) + ); let regressing = second_round_candidate( &model, @@ -1104,6 +1142,10 @@ mod tests { model.record_committed_anchor(regressing_vertex), Err(CertifiedProjectionError::FrontierRegressesCommitted { authority: 0, .. }) )); + assert!( + !model + .committed_frontier_dominates(model.effective_frontier(regressing_vertex).unwrap()) + ); } #[test] diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index e12441e7..5caa8c7a 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -24,10 +24,7 @@ use crate::{ CarrierHeaderV1Args, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, carrier_genesis_reference, - journal::{ - IngressProvenanceV1, JournalErrorV1, JournalEventV1, ValidatedJournalBatchV1, - WriteAheadJournalV1, - }, + journal::{IngressProvenanceV1, JournalErrorV1, JournalEventV1, WriteAheadJournalV1}, model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, projection::{ CertifiedProjectionError, CertifiedProjectionModel, LeaderSlotV1, ProjectionDecisionV1, @@ -226,6 +223,18 @@ pub(crate) enum ShadowDeliveryComparisonV1 { }, } +/// Deterministic application output unlocked by one newly committed clean +/// projected anchor. Carrier references remain available for audit while the +/// application headers are already deduplicated and sorted by their carrier +/// position in the exact committed frontier delta. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CommittedFrontierDeltaV1 { + pub(crate) anchor: ConsensusVertexReference, + pub(crate) frontier: Vec>, + pub(crate) carriers: Vec, + pub(crate) applications: Vec, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct ShadowOpenReportV1 { replayed_batches: u64, @@ -315,8 +324,7 @@ pub(crate) enum ShadowErrorV1 { }, MissingOutboundCandidate(BlockReference), MissingDeliveredCandidate(BlockReference), - PostDurabilityCommit(ModelError), - PostDurabilityJournal(JournalErrorV1), + PostModelJournal(JournalErrorV1), Projection(CertifiedProjectionError), Poisoned, } @@ -382,13 +390,9 @@ impl fmt::Display for ShadowErrorV1 { Self::MissingDeliveredCandidate(reference) => { write!(formatter, "missing delivered shadow candidate {reference}") } - Self::PostDurabilityCommit(error) => write!( + Self::PostModelJournal(error) => write!( formatter, - "shadow model commit failed after WAL durability: {error}" - ), - Self::PostDurabilityJournal(error) => write!( - formatter, - "shadow journal commit failed after WAL durability: {error}" + "shadow journal validation failed after the unpublished model transition: {error}" ), Self::Projection(error) => write!(formatter, "{error}"), Self::Poisoned => formatter.write_str("shadow core is poisoned"), @@ -402,8 +406,8 @@ impl Error for ShadowErrorV1 { Self::Wal(error) => Some(error), Self::Codec(error) => Some(error), Self::Carrier(error) => Some(error), - Self::Model(error) | Self::PostDurabilityCommit(error) => Some(error), - Self::Journal(error) | Self::PostDurabilityJournal(error) => Some(error), + Self::Model(error) => Some(error), + Self::Journal(error) | Self::PostModelJournal(error) => Some(error), Self::Projection(error) => Some(error), _ => None, } @@ -512,16 +516,23 @@ pub(crate) struct StarfishRbcDagShadowV1 { journal: WriteAheadJournalV1, wal: ShadowWalV1, candidates: BTreeMap, + application_carriers: BTreeMap>, delivered: BTreeSet, authenticated_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, requested_recoveries: BTreeMap>, projection: CertifiedProjectionModel, + pending_projection_candidates: BTreeSet, projection_rejected: BTreeMap, projected_decisions: BTreeSet, + included_applications: BTreeSet, + highest_projected_consensus_round: RoundNumber, + next_undecided_consensus_round: RoundNumber, + next_local_consensus_round: RoundNumber, pending_projected_vertices: Vec, pending_projection_decisions: Vec, + pending_committed_frontiers: Vec, poisoned: bool, } @@ -558,7 +569,8 @@ impl StarfishRbcDagShadowV1 { let replayed_batches = recovery.batch_count(); let discarded_tail_bytes = recovery.discarded_tail_bytes(); - let model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; + let mut model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; + model.enable_intrinsic_empty_data_availability(); let projection = CertifiedProjectionModel::from_committee_context(committee.clone()); let journal = WriteAheadJournalV1::new(context, own_authority); let mut core = Self { @@ -570,16 +582,23 @@ impl StarfishRbcDagShadowV1 { journal, wal, candidates: BTreeMap::new(), + application_carriers: BTreeMap::new(), delivered: BTreeSet::new(), authenticated_slots: BTreeMap::new(), ordinarily_retained_slots: BTreeMap::new(), slot_candidates: BTreeMap::new(), requested_recoveries: BTreeMap::new(), projection, + pending_projection_candidates: BTreeSet::new(), projection_rejected: BTreeMap::new(), projected_decisions: BTreeSet::new(), + included_applications: BTreeSet::new(), + highest_projected_consensus_round: 0, + next_undecided_consensus_round: 1, + next_local_consensus_round: 1, pending_projected_vertices: Vec::new(), pending_projection_decisions: Vec::new(), + pending_committed_frontiers: Vec::new(), poisoned: false, }; @@ -591,6 +610,7 @@ impl StarfishRbcDagShadowV1 { // not re-emitted as fresh runtime observations after restart. core.pending_projection_decisions.clear(); core.pending_projected_vertices.clear(); + core.pending_committed_frontiers.clear(); let recovery_effects = core .requested_recoveries .iter() @@ -680,6 +700,10 @@ impl StarfishRbcDagShadowV1 { std::mem::take(&mut self.pending_projected_vertices) } + pub(crate) fn drain_committed_frontiers(&mut self) -> Vec { + std::mem::take(&mut self.pending_committed_frontiers) + } + /// Persist the external data-availability predicate for one exact /// application-bearing carrier. Control carriers are available by shape /// and never require this oracle. @@ -699,30 +723,10 @@ impl StarfishRbcDagShadowV1 { } pub(crate) fn application_carriers(&self, application: BlockReference) -> Vec { - self.candidates - .iter() - .filter_map(|(reference, candidate)| { - candidate - .header() - .application_header() - .is_some_and(|header| header.reference() == application) - .then_some(*reference) - }) - .collect() - } - - /// Application headers with the canonical empty commitment need no - /// transaction reconstruction. Their exact carrier bytes are therefore - /// sufficient data-availability evidence once the carrier is retained. - pub(crate) fn intrinsically_available_applications(&self) -> Vec { - self.candidates - .values() - .filter_map(|candidate| candidate.header().application_header()) - .filter(|header| header.transactions_commitment() == TransactionsCommitment::default()) - .map(RbcCanonicalHeader::reference) - .collect::>() - .into_iter() - .collect() + self.application_carriers + .get(&application) + .map(|carriers| carriers.iter().copied().collect()) + .unwrap_or_default() } pub(crate) fn carrier_data_available(&self, reference: BlockReference) -> bool { @@ -918,12 +922,7 @@ impl StarfishRbcDagShadowV1 { } fn next_local_consensus_round(&self) -> RoundNumber { - let snapshot = self.journal.snapshot(); - let mut round: RoundNumber = 1; - while snapshot.consensus_slot(round).is_some() { - round = round.saturating_add(1); - } - round + self.next_local_consensus_round } /// Verify and durably apply an authenticated network envelope for this @@ -1184,20 +1183,25 @@ impl StarfishRbcDagShadowV1 { ) -> Result, ShadowErrorV1> { self.delivered .iter() - .map(|reference| { - let candidate = self - .candidates - .get(reference) - .ok_or(ShadowErrorV1::MissingDeliveredCandidate(*reference))?; - Ok(ShadowDeliveryIdentityV1::new( - candidate.header().author(), - candidate.header().carrier_round(), - candidate.header().transactions_commitment(), - )) - }) + .map(|reference| self.delivery_identity(*reference)) .collect() } + pub(crate) fn delivery_identity( + &self, + reference: BlockReference, + ) -> Result { + let candidate = self + .candidates + .get(&reference) + .ok_or(ShadowErrorV1::MissingDeliveredCandidate(reference))?; + Ok(ShadowDeliveryIdentityV1::new( + candidate.header().author(), + candidate.header().carrier_round(), + candidate.header().transactions_commitment(), + )) + } + /// Exact application headers whose enclosing carriers reached embedded /// RBC delivery. Control-only carrier deliveries are intentionally absent. pub(crate) fn delivered_application_headers( @@ -1206,22 +1210,29 @@ impl StarfishRbcDagShadowV1 { self.delivered .iter() .filter_map(|carrier_reference| { - let candidate = match self.candidates.get(carrier_reference) { - Some(candidate) => candidate, - None => { - return Some(Err(ShadowErrorV1::MissingDeliveredCandidate( - *carrier_reference, - ))); - } - }; - candidate - .header() - .application_header() - .map(|header| Ok((*carrier_reference, header.clone()))) + match self.delivered_application_header(*carrier_reference) { + Ok(Some(application)) => Some(Ok(application)), + Ok(None) => None, + Err(error) => Some(Err(error)), + } }) .collect() } + pub(crate) fn delivered_application_header( + &self, + carrier_reference: BlockReference, + ) -> Result, ShadowErrorV1> { + let candidate = self + .candidates + .get(&carrier_reference) + .ok_or(ShadowErrorV1::MissingDeliveredCandidate(carrier_reference))?; + Ok(candidate + .header() + .application_header() + .map(|header| (carrier_reference, header.clone()))) + } + /// Compare protocol-independent delivery sets. Multiple transaction /// commitments for one `(author, round)` slot make the comparison /// ambiguous instead of being resolved by arrival or reference order. @@ -1261,36 +1272,23 @@ impl StarfishRbcDagShadowV1 { loop { let mut advanced = false; let candidates = self - .candidates + .pending_projection_candidates .iter() - .filter_map(|(reference, candidate)| { - candidate - .header() - .consensus_vertex() - .is_some() - .then_some(*reference) - }) + .copied() .collect::>(); for reference in candidates { - let consensus_round = self - .candidates - .get(&reference) - .and_then(|candidate| candidate.header().consensus_vertex()) - .expect("candidate list contains only consensus vertices") - .consensus_round(); - let vertex_reference = ConsensusVertexReference::new(reference, consensus_round); - if self.projection.is_projected(vertex_reference) - || self.projection_rejected.contains_key(&reference) - { - continue; - } match self.projection.try_project(reference) { Ok(projected) => { + self.pending_projection_candidates.remove(&reference); + self.highest_projected_consensus_round = self + .highest_projected_consensus_round + .max(projected.consensus_round()); self.pending_projected_vertices.push(projected); advanced = true; } Err(error) if projection_error_is_pending(&error) => {} Err(error) => { + self.pending_projection_candidates.remove(&reference); self.projection_rejected.insert(reference, error); } } @@ -1300,32 +1298,125 @@ impl StarfishRbcDagShadowV1 { } } - let highest_round = self - .candidates - .values() - .filter_map(|candidate| candidate.header().consensus_vertex()) - .map(ConsensusVertexV1::consensus_round) - .max() - .unwrap_or_default(); - for round in 1..=highest_round.saturating_sub(2) { - let slot = self.projection.leader_slot(round); - if self - .projected_decisions - .iter() - .any(|decision| projection_decision_slot(*decision) == slot) + self.drive_ordered_committer(self.highest_projected_consensus_round); + } + + fn drive_ordered_committer(&mut self, highest_round: RoundNumber) { + let decidable_round = highest_round.saturating_sub(2); + loop { + while self.next_undecided_consensus_round <= decidable_round + && self.has_projection_decision( + self.projection + .leader_slot(self.next_undecided_consensus_round), + ) { - continue; + self.next_undecided_consensus_round = + self.next_undecided_consensus_round.saturating_add(1); + } + if self.next_undecided_consensus_round > decidable_round { + return; } + let round = self.next_undecided_consensus_round; + let slot = self.projection.leader_slot(round); let Ok(decision) = self.projection.direct_decision(slot) else { - continue; + return; }; - if matches!(decision, ProjectionDecisionV1::Undecided { .. }) { - continue; + match decision { + ProjectionDecisionV1::DirectCommit { leader } => { + self.commit_projected_anchor(leader); + self.record_projection_decision(decision); + self.next_undecided_consensus_round = round.saturating_add(1); + } + ProjectionDecisionV1::DirectSkip { .. } => { + self.record_projection_decision(decision); + self.next_undecided_consensus_round = round.saturating_add(1); + } + ProjectionDecisionV1::Undecided { .. } => { + let first_anchor_round = round.saturating_add(3); + let later_anchor = + (first_anchor_round..=decidable_round).find_map(|candidate| { + let candidate_slot = self.projection.leader_slot(candidate); + match self.projection.direct_decision(candidate_slot).ok()? { + ProjectionDecisionV1::DirectCommit { leader } => Some(leader), + ProjectionDecisionV1::DirectSkip { .. } + | ProjectionDecisionV1::IndirectCommit { .. } + | ProjectionDecisionV1::IndirectSkip { .. } + | ProjectionDecisionV1::Undecided { .. } => None, + } + }); + let Some(anchor) = later_anchor else { + return; + }; + self.commit_projected_anchor(anchor); + let indirect = self + .projection + .indirect_decision(slot, anchor) + .expect("a clean committed later anchor must decide the older slot"); + self.record_projection_decision(indirect); + self.record_projection_decision(ProjectionDecisionV1::DirectCommit { + leader: anchor, + }); + self.next_undecided_consensus_round = round.saturating_add(1); + } + ProjectionDecisionV1::IndirectCommit { .. } + | ProjectionDecisionV1::IndirectSkip { .. } => { + unreachable!("direct decision returned an indirect result") + } } - if self.projected_decisions.insert(decision) { - self.pending_projection_decisions.push(decision); + } + } + + fn has_projection_decision(&self, slot: LeaderSlotV1) -> bool { + self.projected_decisions + .iter() + .any(|decision| projection_decision_slot(*decision) == slot) + } + + fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { + if self.projected_decisions.insert(decision) { + self.pending_projection_decisions.push(decision); + } + } + + fn commit_projected_anchor(&mut self, anchor: ConsensusVertexReference) { + if self.projection.is_committed_anchor(anchor) { + return; + } + let frontier = self + .projection + .effective_frontier(anchor) + .expect("a committed anchor must be clean and projected") + .to_vec(); + if let Err(error) = self.projection.record_committed_anchor(anchor) { + if self.projection.committed_frontier_dominates(&frontier) { + // The leader is logically committed, but a later anchor used + // for an indirect decision has already output its complete + // carrier prefix. Re-emitting it would regress the frontier. + return; } + panic!("ordered clean anchors must be comparable exact frontiers: {error}"); } + let carriers = self + .model + .apply_frontier(&frontier) + .expect("projection and reducer closed prefixes must agree"); + let applications = carriers + .iter() + .filter_map(|reference| { + self.candidates + .get(reference) + .and_then(|candidate| candidate.header().application_header()) + }) + .filter(|header| self.included_applications.insert(header.reference())) + .cloned() + .collect(); + self.pending_committed_frontiers + .push(CommittedFrontierDeltaV1 { + anchor, + frontier, + carriers, + applications, + }); } fn ensure_live(&self) -> Result<(), ShadowErrorV1> { @@ -1369,20 +1460,28 @@ impl StarfishRbcDagShadowV1 { } fn apply_durable(&mut self, input: ShadowInputV1) -> Result, ShadowErrorV1> { - let plan = self.model.plan_input(input.model_input())?; - let records = encode_batch(self.context, self.own_authority, &input, plan.trace())?; - let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; - self.wal.append_batch(&records)?; - let effects = match self.model.commit_plan(plan) { - Ok(effects) => effects, + let (trace, effects) = self.model.apply_input_unpublished(input.model_input())?; + let records = match encode_batch(self.context, self.own_authority, &input, &trace) { + Ok(records) => records, + Err(error) => { + self.poisoned = true; + return Err(error); + } + }; + let journal_events = match journal_transition_events(&self.journal, &input, &trace) { + Ok(events) => events, Err(error) => { self.poisoned = true; - return Err(ShadowErrorV1::PostDurabilityCommit(error)); + return Err(error); } }; - if let Err(error) = self.journal.commit_validated_batch(journal_batch) { + if let Err(error) = self.journal.apply_batch_unpublished(journal_events) { self.poisoned = true; - return Err(ShadowErrorV1::PostDurabilityJournal(error)); + return Err(ShadowErrorV1::PostModelJournal(error)); + } + if let Err(error) = self.wal.append_batch(&records) { + self.poisoned = true; + return Err(error.into()); } self.record_committed_input(&input, &effects); Ok(effects) @@ -1401,15 +1500,14 @@ impl StarfishRbcDagShadowV1 { self.own_authority, self.committee.committee().len(), )?; - let plan = self.model.plan_input(input.model_input())?; - if plan.trace() != recorded_trace { + let (trace, effects) = self.model.apply_input_unpublished(input.model_input())?; + if trace != recorded_trace { return Err(ShadowErrorV1::TraceMismatch { batch_sequence }); } - let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; - let effects = self.model.commit_plan(plan)?; - if let Err(error) = self.journal.commit_validated_batch(journal_batch) { + let journal_events = journal_transition_events(&self.journal, &input, &trace)?; + if let Err(error) = self.journal.apply_batch_unpublished(journal_events) { self.poisoned = true; - return Err(ShadowErrorV1::PostDurabilityJournal(error)); + return Err(ShadowErrorV1::PostModelJournal(error)); } self.record_committed_input(&input, &effects); Ok(effects) @@ -1489,10 +1587,26 @@ impl StarfishRbcDagShadowV1 { if let Some(candidate) = input.candidate().cloned() { let reference = candidate.reference(); let slot = carrier_slot(reference); + if let Some(vertex) = candidate.header().consensus_vertex() { + self.pending_projection_candidates.insert(reference); + if input.is_local() { + self.next_local_consensus_round = self + .next_local_consensus_round + .max(vertex.consensus_round().saturating_add(1)); + } + } + if let Some(application) = candidate.header().application_header() { + self.application_carriers + .entry(application.reference()) + .or_default() + .insert(reference); + } self.projection .stage_carrier(candidate.clone()) .expect("durably validated carrier must match projection committee"); - if candidate.header().application_header().is_none() || input.is_local() { + if candidate.header().transactions_commitment() == TransactionsCommitment::default() + || input.is_local() + { self.projection .mark_data_available(reference) .expect("staged control carrier is available"); @@ -1737,11 +1851,11 @@ fn decode_authentication( Ok(authentication) } -fn validate_journal_transition( +fn journal_transition_events( journal: &WriteAheadJournalV1, input: &ShadowInputV1, trace: &[ModelTraceEvent], -) -> Result { +) -> Result, ShadowErrorV1> { let mut events = Vec::new(); let context = journal.snapshot().context(); match input { @@ -1867,7 +1981,7 @@ fn validate_journal_transition( reference: authenticated.reference(), }); } - journal.validate_batch(events).map_err(Into::into) + Ok(events) } fn encode_batch( @@ -2917,6 +3031,64 @@ mod tests { assert!(!round_is_stale(66, 2)); } + #[test] + fn rejected_far_future_ingress_does_not_poison_the_durable_actor() { + let mut network = TestNetwork::new(); + let author = 1; + let previous = |authority: AuthorityIndex| BlockReference { + authority, + round: 5, + digest: BlockDigest::from([0x90 + authority as u8; 32]), + }; + let candidate = CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 6, + own_prev: previous(author), + weak_parents: [0, 2].into_iter().map(previous).collect(), + transactions_commitment: TransactionsCommitment::default(), + application_header: None, + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: 1, + }, + &network.committee, + ) + .unwrap(); + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: author, + keys: &network.keyrings[author as usize], + }, + ) + .unwrap(); + + assert!(matches!( + network.nodes[0].receive_or_retain_from_peer( + &candidate.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + author, + ), + Err(ShadowErrorV1::Model( + ModelError::FutureCarrierOutsideBuffer { + current: 1, + maximum: 5, + actual: 6, + } + )) + )); + assert_eq!(network.nodes[0].wal_counts(), (0, 0)); + network.nodes[0] + .create_local_control_heartbeat(2, true) + .expect("a preflight rejection must leave the actor live"); + assert_eq!(network.nodes[0].wal_counts().0, 1); + } + #[test] fn caller_supplied_relay_provenance_must_be_canonical_for_the_author() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index acd0fbeb..7aff59da 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -40,9 +40,9 @@ use crate::{ storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ - ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, - ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, ShadowOpenReportV1, - ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, + CommittedFrontierDeltaV1, ShadowAuthorizerV1, ShadowDeliveryComparisonV1, + ShadowDeliveryIdentityV1, ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, + ShadowOpenReportV1, ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, }, types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, }; @@ -408,6 +408,7 @@ pub(crate) enum ShadowServiceEventV1 { }, VertexProjected(ConsensusVertexReference), LeaderDecided(ProjectionDecisionV1), + FrontierCommitted(CommittedFrontierDeltaV1), Comparison(ShadowDeliveryComparisonV1), Input { kind: &'static str, @@ -983,7 +984,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( invalidated_by_overload: actor_invalidated_by_overload, pending_local, assigned_applications, - available_applications: BTreeSet::new(), + pending_data_availability: BTreeSet::new(), pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), @@ -1114,7 +1115,7 @@ struct ShadowServiceStateV1 { invalidated_by_overload: Arc>>, pending_local: BTreeMap, assigned_applications: BTreeSet, - available_applications: BTreeSet, + pending_data_availability: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, @@ -1307,6 +1308,15 @@ impl ShadowServiceStateV1 { let carrier_round_advanced = effects .iter() .any(|effect| matches!(effect, ModelEffect::CarrierRoundAdvanced(_))); + let newly_delivered = effects + .iter() + .filter_map(|effect| match effect { + ModelEffect::Delivered(reference) => Some(*reference), + ModelEffect::NeedCarrier { .. } + | ModelEffect::PrefixAdvanced { .. } + | ModelEffect::CarrierRoundAdvanced(_) => None, + }) + .collect::>(); if carrier_round_advanced { self.sync_catch_up = std::mem::take(&mut self.sync_used_in_open_round); } @@ -1333,7 +1343,7 @@ impl ShadowServiceStateV1 { self.pending_recovery.len(), )); self.reconcile_data_availability(); - self.report_new_shadow_deliveries(); + self.report_new_shadow_deliveries(&newly_delivered); self.report_projection_progress(); self.flush_carrier_sync_requests(false); self.emit_clock_state(); @@ -1343,9 +1353,14 @@ impl ShadowServiceStateV1 { if !self.mode.is_autonomous() { return; } - let mut desired = self.desired_available_applications.lock().clone(); - desired.extend(self.core.intrinsically_available_applications()); - for application in desired { + let incoming = std::mem::take(&mut *self.desired_available_applications.lock()); + self.pending_data_availability.extend(incoming); + let pending = self + .pending_data_availability + .iter() + .copied() + .collect::>(); + for application in pending { let carriers = self.core.application_carriers(application); if carriers.is_empty() { continue; @@ -1370,7 +1385,7 @@ impl ShadowServiceStateV1 { .iter() .all(|carrier| self.core.carrier_data_available(*carrier)) { - self.available_applications.insert(application); + self.pending_data_availability.remove(&application); } } } @@ -1382,6 +1397,9 @@ impl ShadowServiceStateV1 { for decision in self.core.drain_projection_decisions() { self.emit(ShadowServiceEventV1::LeaderDecided(decision)); } + for delta in self.core.drain_committed_frontiers() { + self.emit(ShadowServiceEventV1::FrontierCommitted(delta)); + } } fn try_create_autonomous_carrier(&mut self, allow_no_vote: bool) { @@ -1864,41 +1882,42 @@ impl ShadowServiceStateV1 { } } - fn report_new_shadow_deliveries(&mut self) { - let identities: BTreeSet<_> = match self.core.delivered_identities() { - Ok(identities) => identities.into_iter().collect(), - Err(error) => { - self.reject(None, error); - return; + fn report_new_shadow_deliveries(&mut self, references: &[BlockReference]) { + for reference in references { + let identity = match self.core.delivery_identity(*reference) { + Ok(identity) => identity, + Err(error) => { + self.reject(None, error); + return; + } + }; + if !self.reported_shadow_deliveries.insert(identity) { + continue; } - }; - let new_identities = identities - .difference(&self.reported_shadow_deliveries) - .copied() - .collect::>(); - self.reported_shadow_deliveries = identities; - for identity in &new_identities { - let slot = delivery_slot(identity); + let slot = delivery_slot(&identity); if !self.mode.is_autonomous() { self.comparison_backlog.observe_epoch_shadow(slot); } - self.emit(ShadowServiceEventV1::Delivered(*identity)); + self.emit(ShadowServiceEventV1::Delivered(identity)); self.emit_slot_comparison(slot); self.emit_comparison_backlog(); - } - let applications = match self.core.delivered_application_headers() { - Ok(applications) => applications, - Err(error) => { - self.reject(None, error); - return; - } - }; - for (carrier, header) in applications { - if self - .reported_application_deliveries - .insert(header.reference()) - { - self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header }); + match self.core.delivered_application_header(*reference) { + Ok(Some((carrier, header))) => { + if self + .reported_application_deliveries + .insert(header.reference()) + { + self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { + carrier, + header, + }); + } + } + Ok(None) => {} + Err(error) => { + self.reject(None, error); + return; + } } } } @@ -2247,10 +2266,7 @@ fn validate_wire_size( fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { matches!( error, - ShadowErrorV1::Wal(_) - | ShadowErrorV1::PostDurabilityCommit(_) - | ShadowErrorV1::PostDurabilityJournal(_) - | ShadowErrorV1::Poisoned + ShadowErrorV1::Wal(_) | ShadowErrorV1::PostModelJournal(_) | ShadowErrorV1::Poisoned ) } @@ -2464,6 +2480,7 @@ mod tests { open_rounds: &mut [RoundNumber], deliveries: &mut [usize], application_deliveries: &mut [BTreeSet], + committed_frontiers: &mut [Vec], sync_requests: &mut usize, projected_vertices: &mut usize, projected_decisions: &mut usize, @@ -2531,6 +2548,9 @@ mod tests { ShadowServiceEventV1::LeaderDecided(_) => { *projected_decisions = projected_decisions.saturating_add(1); } + ShadowServiceEventV1::FrontierCommitted(delta) => { + committed_frontiers[sender].push(delta); + } ShadowServiceEventV1::Rejected { error, .. } if error.contains("FutureCarrierOutsideBuffer") || error.contains("unexpected shadow response") => {} @@ -2789,6 +2809,7 @@ mod tests { let mut open_rounds = vec![1; n]; let mut deliveries = vec![0; n]; let mut application_deliveries = vec![BTreeSet::new(); n]; + let mut committed_frontiers = vec![Vec::new(); n]; let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; @@ -2809,6 +2830,7 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, + &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, @@ -2823,6 +2845,12 @@ mod tests { ); assert!(open_rounds.iter().all(|round| *round >= 19)); assert!(projected_vertices >= n * 3); + assert!( + committed_frontiers + .iter() + .all(|commits| !commits.is_empty()), + "every node must commit at least one certified frontier: {committed_frontiers:?}" + ); assert!( projected_decisions > 0, "clean projection did not decide: vertices={projected_vertices}, rounds={open_rounds:?}" @@ -2905,6 +2933,7 @@ mod tests { let mut open_rounds = vec![1; N]; let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; + let mut committed_frontiers = vec![Vec::new(); N]; let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; @@ -2914,12 +2943,34 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, + &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, 5, ) .await; + // The first directly committed consensus frontier may predate the + // round-one application deliveries. Advance enough certified carrier + // rounds for a later committed frontier to include that closed prefix. + for fixed_round in 5..=18 { + for handle in &handles { + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + } + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut application_deliveries, + &mut committed_frontiers, + &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, + fixed_round + 1, + ) + .await; + } assert!( application_deliveries @@ -2929,13 +2980,36 @@ mod tests { ); assert!( open_rounds.iter().all(|round| *round >= 5), - "application-critical phase carriers must not wait for a heartbeat tick" + "the carrier DAG must keep advancing after application delivery" ); assert_eq!(sync_requests, 0); assert!( projected_vertices >= N, "empty embedded applications must not stall clean projection" ); + let committed_applications = committed_frontiers + .iter() + .map(|commits| { + commits + .iter() + .flat_map(|delta| delta.applications.iter().map(RbcCanonicalHeader::reference)) + .collect::>() + }) + .collect::>(); + assert!( + committed_applications + .iter() + .all(|applications| applications == &committed_applications[0]), + "equal certified frontiers must output byte-identical application order: {committed_applications:?}" + ); + assert_eq!( + committed_applications[0] + .iter() + .copied() + .collect::>(), + expected, + "the committed frontier closure must output every exact application once; commits={committed_frontiers:?}" + ); drop(events); for handle in &handles { @@ -3088,6 +3162,7 @@ mod tests { let mut open_rounds = vec![1; N]; let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; + let mut committed_frontiers = vec![Vec::new(); N]; let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; @@ -3100,6 +3175,7 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, + &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, @@ -3153,6 +3229,7 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, + &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 59e8de2c..cf454cb6 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -20,6 +20,7 @@ use crate::{ runtime::timestamp_utc, sailfish_service::SailfishServiceMessage, starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, + starfish_rbc_dag_shadow::CommittedFrontierDeltaV1, starfish_rbc_dag_shadow_service::StarfishRbcDagShadowServiceHandleV1, starfish_rbc_service::{RbcLocalHeader, RbcServiceHandle}, types::{ @@ -69,6 +70,7 @@ pub struct Syncer { sailfish_tx: Option>, starfish_rbc_service: Option, starfish_rbc_dag_shadow_service: Option, + rbc_dag_frontier_authority: bool, } pub trait SyncerSignals: Send + Sync { @@ -83,6 +85,13 @@ pub trait CommitObserver: Send + Sync { committed_leaders: Vec<(Data, Option)>, ) -> Vec; + fn handle_rbc_dag_commit( + &mut self, + dag_state: &DagState, + anchor: BlockReference, + applications: &[BlockReference], + ) -> Vec; + fn recover_committed( &mut self, committed: AHashSet, @@ -94,7 +103,7 @@ pub trait CommitObserver: Send + Sync { impl Syncer { pub fn new( - core: Core, + mut core: Core, signals: S, commit_observer: C, metrics: Arc, @@ -102,7 +111,11 @@ impl Syncer { sailfish_tx: Option>, starfish_rbc_service: Option, starfish_rbc_dag_shadow_service: Option, + rbc_dag_frontier_authority: bool, ) -> Self { + if rbc_dag_frontier_authority { + core.enable_rbc_dag_application_production(); + } let committee_size = core.committee().len(); let own_stake = core .committee() @@ -123,6 +136,7 @@ impl Syncer { sailfish_tx, starfish_rbc_service, starfish_rbc_dag_shadow_service, + rbc_dag_frontier_authority, } } @@ -239,6 +253,28 @@ impl Syncer { } } + /// Sequence one exact deterministic carrier-frontier delta. In M7 this is + /// the sole application-ordering authority; the legacy Starfish committer + /// remains disabled in this mode. + pub fn apply_starfish_rbc_dag_frontier(&mut self, delta: CommittedFrontierDeltaV1) { + assert!( + self.rbc_dag_frontier_authority, + "RBC-DAG frontier output requires the explicit authority mode" + ); + let applications = delta + .applications + .iter() + .map(RbcCanonicalHeader::reference) + .collect::>(); + let committed = self.commit_observer.handle_rbc_dag_commit( + self.core.dag_state(), + delta.anchor.carrier(), + &applications, + ); + self.core.handle_rbc_dag_committed_delta(committed); + self.try_new_block(BlockCreationReason::PostCommit); + } + /// Store a Sailfish++ timeout certificate in DagState and retry block /// creation (a TC may unblock block creation for the next round). pub fn apply_timeout_cert(&mut self, cert: SailfishTimeoutCert) { @@ -477,6 +513,9 @@ impl Syncer { } pub fn try_new_commit(&mut self) { + if self.rbc_dag_frontier_authority { + return; + } let (newly_committed, any_decided) = self.core.try_commit(); let utc_now = timestamp_utc(); if !newly_committed.is_empty() { @@ -563,6 +602,15 @@ mod tests { Vec::new() } + fn handle_rbc_dag_commit( + &mut self, + _dag_state: &DagState, + _anchor: BlockReference, + _applications: &[BlockReference], + ) -> Vec { + Vec::new() + } + fn recover_committed( &mut self, _committed: AHashSet, @@ -665,6 +713,7 @@ mod tests { None, None, None, + false, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); @@ -764,6 +813,7 @@ mod tests { None, None, None, + false, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 818b9fba..43e202fd 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -355,7 +355,14 @@ mod smoke_tests { let route = prometheus::METRICS_ROUTE; let res = reqwest::get(format! {"http://{address}{route}"}).await?; let string = res.text().await?; - let commit = string.contains("committed_leaders_total"); + let commit = string.lines().any(|line| { + line.starts_with("commit_index") + && line + .split_whitespace() + .last() + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value > 0.0) + }); Ok(commit) } @@ -577,12 +584,38 @@ mod smoke_tests { let timeout_multiplier = if consensus == "starfish-rbc" { 20 } else { 5 }; let timeout = config::param_defaults::default_leader_timeout() * timeout_multiplier; - tokio::select! { - _ = await_for_commits(addresses) => (), - _ = time::sleep(timeout) => panic!( - "[{consensus}] Failed to gather commits \ - within a few timeouts" - ), + if tokio::time::timeout(timeout, await_for_commits(addresses)) + .await + .is_err() + { + let state = validators + .iter() + .map(|validator| { + let metrics = validator.metrics(); + ( + metrics.commit_index.get(), + metrics.starfish_rbc_dag_shadow_carrier_round.get(), + metrics.starfish_rbc_dag_projected_vertices_total.get(), + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get(), + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get(), + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get(), + ) + }) + .collect::>(); + panic!( + "[{consensus}] Failed to gather commits within a few timeouts; \ + per-node (commit index, carrier round, projected, direct commits, frontiers, \ + frontier applications)={state:?}" + ); } if autonomous_clock { @@ -617,7 +650,17 @@ mod smoke_tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get() - > 0) + > 0 + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get() + > 0 + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get() + > 0) && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 }) { break; diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index b2106bc3..59a9ae77 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,7 +1,7 @@ # Starfish-RBC-DAG protocol design -Status: milestone-six certified-projection prototype; frontier output and end-to-end safety/liveness -proof remain incomplete +Status: milestone-seven committed-frontier output prototype; end-to-end safety/liveness proof, +proof-safe retirement, and full validator recovery remain incomplete The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not implemented. The staged prototype runs under `starfish-rbc`: direct-header comparison uses @@ -12,9 +12,9 @@ only application-header certification authority with but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add `--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Autonomous carriers now create durably locked logical consensus vertices and the clean projection produces -Starfish commit/skip decisions. Deterministic frontier output still uses the existing Starfish DAG -until milestone seven. The eventual protocol is new, not a transport option or a version-two alias -for `starfish-rbc`. +Starfish commit/skip decisions. In embedded-authority mode, committed projected anchors now release +deterministic exact carrier-frontier deltas and the legacy Starfish committer is disabled. The +eventual protocol is new, not a transport option or a version-two alias for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -57,12 +57,14 @@ the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application ca encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. Milestone six adds independently numbered consensus vertices, quorum strong parents, objective Vote/NoVote choices, exact contiguous delivery frontiers, durable local consensus locks, and a live committer -that consumes only RBC-delivered, data-available projected vertices. +that consumes only RBC-delivered, data-available projected vertices. Milestone seven persists and +reconstructs the corresponding anchor/frontier state, applies exact closed-prefix deltas, and makes +those deltas the sole application output authority. -The current authoritative mode changes header certification and produces certified carrier-DAG -leader decisions. Direct INIT is still the application-payload transport and is not a certification -vote. The legacy Starfish DAG remains only as the temporary application-output scaffold; replacing -its commit/output path with deterministic committed frontier deltas is milestone seven. +The current authoritative mode changes header certification and application ordering. Direct INIT +is still the application-payload transport and is not a certification vote. Direct ECHO/READY and +the legacy Starfish committer cannot certify or output applications in this mode; only the +carrier-DAG projection's committed frontier deltas can do so. Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode requires an identical recovered direct-header history, control-only autonomous history reopens @@ -93,8 +95,10 @@ therefore forbidden until the worker has exited; process exit remains safe. A pr same-process restart path needs an operating-system file lock or a fully cancellable storage task. The shadow runtime is not a proof or a production performance implementation. Its default -crash-safe profile intentionally fsyncs every accepted transition, and its reference reducer clones -retained model/journal history. A separate explicit benchmark profile writes the same ordered, +crash-safe profile intentionally fsyncs every accepted transition. The live fail-stop reducer now +applies preflighted transitions and journal deltas in place, then exposes effects only after WAL +append, avoiding the former full-history clone on every carrier. It still retains unbounded run +history and performs synchronous reducer/storage work. A separate explicit benchmark profile writes the same ordered, checksummed frames but syncs them only on clean shutdown; it reports appended and durable records separately and makes no crash-safety claim. This removes the known persistence observer effect without changing the protocol reducer. The runtime also uses a fixed unsolicited-retention window @@ -803,8 +807,9 @@ per peer. Requested historical slots remain recoverable beyond the benchmark-onl retention window. Autonomous benchmark validity is separate from delivery comparison validity. -`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and heartbeat counters must progress, -the carrier round and embedded-RBC delivery count must advance during the measured interval, +`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and local-carrier counters +must progress, an idle heartbeat must have been observed, the carrier round and embedded-RBC +delivery count must advance during the measured interval, recovery must drain, and the reported clock-state/backlog and cross-node skew must remain within the configured empirical guards. These checks establish that the observational carrier plane stayed live and bounded; they are not a partial-synchrony proof. @@ -885,7 +890,10 @@ minimum it must cover: proactive rounds, exact-slot synchronization with idempotent late responses and per-peer rate limiting, multi-round convergence after a validator falls behind, control-only WAL reopen, distinct authentication namespace, and an integration check that direct Starfish-RBC continues - committing while the observational carrier clock advances. + committing while the observational carrier clock advances; and +- composed frontier-authority runs in which every exact application header is embedded-RBC + delivered, all honest nodes release the same deterministic application order without duplicates, + the legacy committer is disabled, and the frontier/application/WAL progress gates remain valid. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -930,6 +938,7 @@ samples. | Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | | Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | +| Frontier output authority (milestone seven) | VALID 10/10 | 950.25 | 2,020.9 ms | 2,082.6 ms | 0.70 MB/s | | Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted @@ -954,6 +963,15 @@ pays for both and its 5.1/8.7-second latency is a red flag rather than a protoco seven must make committed frontier deltas the sole ordering/output path before latency is compared as the complete RBC-DAG protocol. +Milestone seven removes that legacy output gate. Its valid run reached carrier round 795 at every +validator, delivered 79,035 application carriers, released 77,870 exact applications through 1,880 +committed frontiers, projected 19,000 vertices, and ended with zero pending recovery. The in-place +fail-stop reducer, incremental projection indexes, and event-local delivery/data-availability paths +also remove the prototype's history-wide hot-path scans. Block/E2E latency fell by 60.4%/75.9% from +milestone six while throughput recovered to 950.25 tx/s. The remaining 2.02/2.08-second latency is +not the target: follow-up profiling must shorten the certified consensus-round/frontier pipeline +toward the roughly 600 ms unsafe Starfish-MAC reference without weakening RBC or frontier safety. + ## 19. Contained implementation milestones Every milestone is committed separately. @@ -983,8 +1001,11 @@ Every milestone is committed separately. vertices, quorum strong parents, explicit timeout-bound leader choices, contiguous exact delivery frontiers, durable slot/choice locks, and a live clean-only direct committer. Malformed optional vertices do not poison their enclosing carrier. -7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct - prefixes and anchors, and add late-node and crash/restart tests. +7. **Frontier linearizer and recovery (implemented within the actor's fail-stop scope):** commit + deterministic frontier deltas, reconstruct prefixes, decisions, and anchors from the ordered + WAL, disable the legacy application committer, and output exact application references once. + Full-validator crash recovery and proof-safe late-node state transfer remain deferred because + the direct payload-transport baseline does not yet persist its own proof-critical RBC state. 8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, signature Starfish variants, and Sailfish++ before attempting tree dissemination. 9. **Tree dissemination:** distribute vector sub-bundles with redundant routing and a direct timeout From 63d0bc23b7c17c0652d2ba7b7d9d2ff47319e589 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:56:53 +0200 Subject: [PATCH 36/62] Implement standalone Starfish RBC-DAG testbed --- Cargo.lock | 1 + README.md | 65 +- crates/orchestrator/src/measurements.rs | 63 +- crates/starfish-core/src/block_handler.rs | 57 +- crates/starfish-core/src/config.rs | 10 +- crates/starfish-core/src/core.rs | 742 ++- .../starfish-core/src/core_thread/spawned.rs | 136 +- crates/starfish-core/src/dag_state.rs | 37 + crates/starfish-core/src/metrics.rs | 1478 ++++- crates/starfish-core/src/net_sync.rs | 3059 +++++++++- crates/starfish-core/src/network.rs | 1283 +++- crates/starfish-core/src/rocks_store.rs | 208 +- .../src/starfish_rbc_dag/journal.rs | 648 +- .../starfish-core/src/starfish_rbc_dag/mod.rs | 96 +- .../src/starfish_rbc_dag/model.rs | 1065 +++- .../src/starfish_rbc_dag/projection.rs | 1308 +++- .../src/starfish_rbc_dag_shadow.rs | 2081 ++++++- .../src/starfish_rbc_dag_shadow_service.rs | 5407 +++++++++++++++-- crates/starfish-core/src/stat.rs | 10 + crates/starfish-core/src/store.rs | 290 +- crates/starfish-core/src/syncer.rs | 631 +- crates/starfish-core/src/tidehunter_store.rs | 201 +- .../src/transactions_generator.rs | 448 +- crates/starfish-core/src/validator.rs | 123 +- crates/starfish/Cargo.toml | 1 + crates/starfish/src/main.rs | 765 ++- docs/starfish-rbc-dag-protocol.md | 729 ++- 27 files changed, 19006 insertions(+), 1936 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc80648f..d586ec61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,6 +3584,7 @@ dependencies = [ "clap", "color-eyre", "eyre", + "futures", "prettytable-rs", "starfish-core", "tokio", diff --git a/README.md b/README.md index 855c189c..b39e91ba 100644 --- a/README.md +++ b/README.md @@ -48,30 +48,44 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG -while keeping certified Starfish consensus and ordering in a separate logical projection. Its -canonical types, deterministic models, crash journal, comparison shadow, autonomous carrier clock, -authoritative embedded-RBC path, and certified logical consensus projection are implemented. Run -the comparison shadow with -`--consensus starfish-rbc --starfish-rbc-dag-shadow`; add -`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact -application headers in version-two carriers and make embedded ECHO/READY/delivery their sole -certification authority. Direct INIT remains payload transport, but direct ECHO/READY cannot clean -blocks in that mode. Committed projected anchors now release deterministic carrier-frontier deltas, -and those deltas are the sole application ordering/output authority; the legacy Starfish committer -is disabled. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for -Starfish-RBC by default); application and encodable phase carriers are emitted immediately. - -Autonomous carriers now embed durably locked consensus vertices with quorum strong parents, -explicit Vote/NoVote choices, and exact delivery frontiers. Only RBC-delivered, data-available, -prefix-closed vertices enter the projection or its leader decisions. Frontier output retains exact -application references and is rebuilt from the ordered WAL on actor reopen. Full validator crash -recovery and proof-safe late-node state transfer remain outside this milestone. Shadow traffic shares the -validator's network socket and bandwidth, and deployment requires a homogeneous new-binary -committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; -`--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean -shutdown and therefore forfeits crash safety. Full validator crash recovery also remains out of -scope. See the [protocol design](docs/starfish-rbc-dag-protocol.md). +**Starfish-RBC-DAG** is the standalone follow-up that carries authentication, reliable-broadcast +control, application headers, and logical Starfish vertices in one optimistic carrier DAG. Run the +comparison plane with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; add +`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to make that carrier +plane the sole reliable-broadcast, consensus, ordering, and output authority. In this mode the direct +Starfish-RBC service is not started: direct INIT/phase/header-recovery messages, generic block +batches, and legacy parent/transaction pulls cannot certify or order an application. Application +bytes travel with the carrier envelope when available, or through the dedicated RBC-DAG payload +request/response path, and are accepted only after commitment verification. + +The implemented MAC-vector RBC uses four embedded phases. For target-author stake `a` and total +stake `W`, ECHO, VOTE, and ACK exclude the target author; weighted thresholds `M`, `C`, and `O` +drive VOTE, ACK/READY convergence, and authoritative optimistic delivery. Reaching `O` ECHO stake +is sufficient for the fast delivery latch, while `Q = W - floor((W - 1) / 3)` READY stake records a +separate slower certification latch. If `a > floor((W - 1) / 3)`, the fault model makes the author +honest and receiver-authenticated exact content can take the fast latch directly. All four phases +use per-sender and local slot-global locks, and threshold evidence without content triggers exact +carrier recovery before a local follow-up is exposed. The full definitions and safety boundary are +in the [protocol design](docs/starfish-rbc-dag-protocol.md). + +Autonomous carriers embed durably locked consensus vertices with quorum strong parents, explicit +Vote/NoVote choices, and exact delivery frontiers. Authoritative delivery alone is not application +data availability: projection and output wait until Core has materialized the concrete application +block and its committed payload is available. Each committed anchor is componentwise joined into a +cumulative committed frontier; only the new exact, data-available prefix delta is released, and +the legacy Starfish committer is disabled. The prototype admits at most two carrier rounds ahead, +retains canonical unsolicited carrier content up to 64 rounds ahead, and offers rate-limited +single-slot exact synchronization rather than checkpoint or proof-safe late-node state transfer. +Its current authoritative journal uses the V4 autonomous WAL namespace with `SRD5` raw records so +older traces cannot be reinterpreted under the optimistic-delivery rules. + +The default WAL syncs every transition. `--starfish-rbc-dag-shadow-buffered-wal` preserves ordered +frames but syncs only on clean shutdown and is not crash-safe. Actor replay covers the state +explicitly documented in the protocol design; full validator crash recovery, bounded checkpoint +transfer, and proof-safe state retirement are not claimed. Shadow traffic shares the validator's +network socket and bandwidth, and deployment requires a homogeneous new-binary committee. Idle +carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for Starfish-RBC by default); +application and encodable phase carriers are emitted immediately. For a direct-header shadow comparison, `starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded observational path was disabled or shed work and the comparison must be discarded. Healthy live @@ -85,7 +99,8 @@ supports at most 60 validators in mirror mode and 20 in autonomous mode. A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared -600 ms leader/idle-carrier timeout. +600 ms leader/idle-carrier timeout. These milestone rows predate the current four-phase V4 +authority model and are retained as historical measurements. | Profile | Verdict | TPS | Block latency | E2E latency | Outbound BW | |---|---:|---:|---:|---:|---:| diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 51af511c..ad68be75 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -16,10 +16,11 @@ use prettytable::{Table, row}; use prometheus_parse::Scrape; use serde::{Deserialize, Serialize}; use starfish_core::metrics::{ - STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR, STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR, STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, + starfish_rbc_dag_autonomous_buffered_capacity_bound, + starfish_rbc_dag_autonomous_buffered_settled_bound, }; use crate::{ @@ -752,6 +753,12 @@ impl MeasurementsCollection { .is_some_and(|measurement| measurement.scalar > minimum) } + fn latest_scalar_at_most(&self, label: &str, scraper_id: ScraperId, maximum: f64) -> bool { + self.scraper_series(label, scraper_id) + .and_then(|series| series.last()) + .is_some_and(|measurement| measurement.scalar >= 0.0 && measurement.scalar <= maximum) + } + fn scalar_gauge_increased(&self, label: &str, scraper_id: ScraperId) -> bool { self.active_window_series(label, scraper_id) .is_some_and(|series| { @@ -1179,8 +1186,11 @@ impl MeasurementsCollection { }; let autonomous_phase_backlog_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let autonomous_buffered_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR - .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); + let committee_size = i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX); + let autonomous_buffered_capacity_bound = + starfish_rbc_dag_autonomous_buffered_capacity_bound(committee_size); + let autonomous_buffered_settled_bound = + starfish_rbc_dag_autonomous_buffered_settled_bound(committee_size); let every_autonomous_scraper_has_progress = shadow_autonomous_clock_valid_scrapers .iter() .all(|scraper_id| { @@ -1247,7 +1257,12 @@ impl MeasurementsCollection { && self.gauge_always_at_most( "starfish_rbc_dag_shadow_buffered_authenticated", *scraper_id, - autonomous_buffered_bound as f64, + autonomous_buffered_capacity_bound as f64, + ) + && self.latest_scalar_at_most( + "starfish_rbc_dag_shadow_buffered_authenticated", + *scraper_id, + autonomous_buffered_settled_bound as f64, ) }); let shadow_autonomous_clock_valid = shadow_autonomous_clock_enabled @@ -1996,10 +2011,13 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 #[test] fn autonomous_clock_has_a_distinct_sticky_summary() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 1, 2, 7, 1, 5); - add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 8, 1, 2, 6, 1, 6); + // Catch-up may transiently use most of the 62 retained slots per + // remote author. That is safe as long as the final healthy tail + // settles to the tighter round-skew-derived bound. + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 1, 2, 7, 50, 5); + add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 8, 1, 2, 6, 60, 6); add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 3, 2, 7, 2, 10); - add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 10, 4, 2, 6, 3, 11); + add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 10, 4, 2, 6, 2, 11); let summary = collection.benchmark_run_summary(); assert!(!summary.shadow_comparison_enabled); @@ -2014,7 +2032,7 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert_eq!(summary.shadow_autonomous_clock_admitted_stake_min, 6); assert_eq!( summary.shadow_autonomous_clock_buffered_authenticated_total, - 5 + 4 ); assert_eq!(summary.shadow_wal_durable_records, 21); @@ -2026,6 +2044,35 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert!(!summary.shadow_autonomous_clock_valid); } + #[test] + fn autonomous_clock_buffer_gate_separates_capacity_from_settled_tail() { + let mut capacity_overflow = + MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); + add_autonomous_clock_snapshot(&mut capacity_overflow, 0, 1.0, 8, 0, 2, 7, 63, 5); + add_autonomous_clock_snapshot(&mut capacity_overflow, 1, 1.0, 8, 0, 2, 7, 0, 5); + add_autonomous_clock_snapshot(&mut capacity_overflow, 0, 1.0, 12, 0, 2, 7, 2, 10); + add_autonomous_clock_snapshot(&mut capacity_overflow, 1, 1.0, 12, 0, 2, 7, 2, 10); + assert!( + !capacity_overflow + .benchmark_run_summary() + .shadow_autonomous_clock_valid, + "one remote author cannot occupy more than the 62-slot retention capacity" + ); + + let mut unsettled_tail = + MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); + add_autonomous_clock_snapshot(&mut unsettled_tail, 0, 1.0, 8, 0, 2, 7, 50, 5); + add_autonomous_clock_snapshot(&mut unsettled_tail, 1, 1.0, 8, 0, 2, 7, 50, 5); + add_autonomous_clock_snapshot(&mut unsettled_tail, 0, 1.0, 12, 0, 2, 7, 3, 10); + add_autonomous_clock_snapshot(&mut unsettled_tail, 1, 1.0, 12, 0, 2, 7, 2, 10); + assert!( + !unsettled_tail + .benchmark_run_summary() + .shadow_autonomous_clock_valid, + "a healthy final scrape must settle to two buffered slots per remote author" + ); + } + #[test] fn autonomous_clock_buffered_wal_uses_appended_progress_without_claiming_durability() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); diff --git a/crates/starfish-core/src/block_handler.rs b/crates/starfish-core/src/block_handler.rs index 76bc4269..28076f35 100644 --- a/crates/starfish-core/src/block_handler.rs +++ b/crates/starfish-core/src/block_handler.rs @@ -32,9 +32,9 @@ const REAL_BLOCK_HANDLER_TXN_SIZE: usize = 512; const REAL_BLOCK_HANDLER_TXN_GEN_STEP: usize = 32; const _: () = assert_constants(); -#[allow(dead_code)] +#[allow(dead_code, clippy::manual_is_multiple_of)] const fn assert_constants() { - if !REAL_BLOCK_HANDLER_TXN_SIZE.is_multiple_of(REAL_BLOCK_HANDLER_TXN_GEN_STEP) { + if REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0 { panic!("REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0") } } @@ -121,13 +121,13 @@ impl RealCommitHandler { } fn transaction_observer(&self, block: Data) { - // Skip every rate-feeding metric outside the active submission - // window. Late commits during warmup or wind-down would otherwise - // skew TPS, the cumulative latency distribution, and the bandwidth- - // efficiency denominator. + // Transaction observations stay open for the bounded post-submission + // drain. No transactions exist before the coordinated start, so this + // records exactly the offered window while allowing the harness to + // distinguish committed-at-cutoff from eventual committed throughput. if !self .metrics - .metrics_active + .transaction_metrics_active .load(std::sync::atomic::Ordering::Relaxed) { return; @@ -144,6 +144,15 @@ impl RealCommitHandler { .transaction_committed_latency_squared_micros .inc_by(latency.as_micros().pow(2) as u64); self.metrics.sequenced_transactions_total.inc(); + let cutoff_micros = self + .metrics + .benchmark_transaction_cutoff_micros + .load(std::sync::atomic::Ordering::Acquire); + if cutoff_micros > 0 + && self.metrics.validator_start.elapsed().as_micros() < cutoff_micros.into() + { + self.metrics.sequenced_transactions_cutoff_total.inc(); + } self.metrics .sequenced_transactions_bytes .inc_by(transaction.as_bytes().len() as u64); @@ -368,37 +377,19 @@ impl CommitObserver for RealCommitHandler { resulted_committed } - fn handle_rbc_dag_commit( - &mut self, - dag_state: &DagState, - anchor: BlockReference, - applications: &[BlockReference], - ) -> Vec { - let blocks = applications - .iter() - .map(|reference| { - let block = dag_state - .get_storage_block(*reference) - .unwrap_or_else(|| panic!("committed RBC-DAG application {reference} missing")); - assert!( - dag_state.is_data_available(reference), - "committed RBC-DAG application {reference} is unavailable" - ); - block - }) - .collect::>(); - let commit = CommittedSubDag::new(anchor, blocks); - self.record_commit_metadata(std::iter::once(&commit)); - for block in &commit.blocks { - if block.round() > 0 { - self.transaction_observer(block.clone()); + fn handle_rbc_dag_commit(&mut self, committed: &[CommittedSubDag]) { + self.record_commit_metadata(committed); + for commit in committed { + for block in &commit.blocks { + if block.round() > 0 { + self.transaction_observer(block.clone()); + } } } - self.sequenced_commit_count += 1; + self.sequenced_commit_count += committed.len(); self.metrics .commit_availability_gap .set((self.committed_count - self.sequenced_commit_count) as i64); - vec![commit] } fn recover_committed( diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index d0c5aebe..d1dd250a 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -406,7 +406,11 @@ impl NodePrivateConfig { pub fn starfish_rbc_dag_autonomous_clock_wal(&self) -> PathBuf { self.storage_path - .join("starfish-rbc-dag-autonomous-clock-v1.wal") + // V4 promotes the proved optimistic ECHO predicate from planning + // into authoritative RBC delivery while retaining the separate + // Q-READY certificate latch. Older traces must not be reinterpreted + // under the stronger output semantics. + .join("starfish-rbc-dag-autonomous-clock-v4.wal") } pub fn starfish_rbc_dag_shadow_buffered_benchmark_wal(&self) -> PathBuf { @@ -416,7 +420,7 @@ impl NodePrivateConfig { pub fn starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal(&self) -> PathBuf { self.storage_path - .join("starfish-rbc-dag-autonomous-clock-buffered-benchmark-v1.wal") + .join("starfish-rbc-dag-autonomous-clock-buffered-benchmark-v3.wal") } } @@ -489,7 +493,7 @@ mod tests { private_config.starfish_rbc_dag_autonomous_clock_wal(), Path::new("benchmark") .join("storage-0") - .join("starfish-rbc-dag-autonomous-clock-v1.wal") + .join("starfish-rbc-dag-autonomous-clock-v4.wal") ); } } diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index 9d701ff3..0a43974a 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -2,7 +2,7 @@ // Modifications Copyright (c) 2025 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{mem, sync::Arc}; +use std::{fmt, mem, sync::Arc}; use ahash::{AHashMap, AHashSet}; use reed_solomon_simd::ReedSolomonEncoder; @@ -31,8 +31,10 @@ use crate::{ encoder::ShardEncoder, metrics::{Metrics, UtilizationTimerVecExt}, runtime::timestamp_utc, + starfish_rbc_dag::ConsensusVertexReference, + starfish_rbc_dag_shadow::RbcDagFrontierRecoveryCursorV1, state::RecoveredState, - store::Store, + store::{RbcDagFrontierReceipt, Store}, types::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, @@ -76,6 +78,103 @@ pub struct Core { /// descriptors only. Their dirty/clean DAG is no longer a consensus or /// output authority, so raw threshold-clock progress may produce them. rbc_dag_application_production: bool, + /// Latest atomically persisted authoritative carrier-frontier cursor. + /// Loaded before the shadow actor opens and advanced only after the + /// commit/receipt storage batch succeeds. + latest_rbc_dag_frontier_cursor: Option, +} + +#[derive(Debug)] +pub(crate) enum RbcDagFrontierApplyError { + StaleSequence { + current_sequence: RoundNumber, + actual_sequence: RoundNumber, + }, + SequenceGap { + expected_sequence: RoundNumber, + actual_sequence: RoundNumber, + }, + ConflictingAnchor { + output_sequence: RoundNumber, + expected: BlockReference, + actual: BlockReference, + }, + ConflictingApplications { + output_sequence: RoundNumber, + anchor: BlockReference, + expected: Vec, + actual: Vec, + }, + ReusedAnchor { + anchor: BlockReference, + previous_sequence: RoundNumber, + actual_sequence: RoundNumber, + }, + MissingApplication(BlockReference), + UnavailableApplication(BlockReference), +} + +impl fmt::Display for RbcDagFrontierApplyError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StaleSequence { + current_sequence, + actual_sequence, + } => write!( + formatter, + "stale RBC-DAG frontier output sequence {actual_sequence}; durable sequence is {current_sequence}" + ), + Self::SequenceGap { + expected_sequence, + actual_sequence, + } => write!( + formatter, + "RBC-DAG frontier output sequence gap: expected {expected_sequence}, got {actual_sequence}" + ), + Self::ConflictingAnchor { + output_sequence, + expected, + actual, + } => write!( + formatter, + "conflicting RBC-DAG frontier anchor at output sequence {output_sequence}: durable {expected}, actual {actual}" + ), + Self::ReusedAnchor { + anchor, + previous_sequence, + actual_sequence, + } => write!( + formatter, + "RBC-DAG carrier anchor {anchor} was reused at output sequence {actual_sequence} after durable sequence {previous_sequence}" + ), + Self::ConflictingApplications { + output_sequence, + anchor, + expected, + actual, + } => write!( + formatter, + "conflicting RBC-DAG frontier applications at output sequence {output_sequence} for anchor {anchor}: durable {expected:?}, actual {actual:?}" + ), + Self::MissingApplication(reference) => { + write!( + formatter, + "committed RBC-DAG application {reference} is missing" + ) + } + Self::UnavailableApplication(reference) => write!( + formatter, + "committed RBC-DAG application {reference} is unavailable" + ), + } + } +} + +impl std::error::Error for RbcDagFrontierApplyError {} + +pub(crate) enum RbcDagFrontierApplyOutcome { + Applied(Vec), + ExactReplay, } #[derive(Debug, Clone)] @@ -104,6 +203,40 @@ impl Core { committed_leaders_count, } = recovered; + let latest_rbc_dag_frontier_receipt = store + .read_latest_rbc_dag_frontier_receipt() + .expect("Failed to read the latest RBC-DAG frontier receipt"); + let latest_rbc_dag_frontier_cursor = latest_rbc_dag_frontier_receipt.map(|receipt| { + assert!( + dag_state.consensus_protocol.is_starfish_rbc(), + "an RBC-DAG frontier receipt cannot be recovered under a non-RBC protocol" + ); + dag_state.restore_rbc_dag_committed_rounds(&receipt.committed_rounds); + let application_references = store + .get_commit(&receipt.carrier_anchor) + .expect("Failed to read the RBC-DAG frontier application commit") + .map(|commit| { + assert!( + !commit.sub_dag.is_empty(), + "a present RBC-DAG frontier application commit must not be empty; control-only frontiers are represented by absence" + ); + assert_eq!( + commit.leader, receipt.carrier_anchor, + "RBC-DAG frontier application commit must be keyed by its carrier anchor" + ); + assert_eq!( + commit.committed_rounds, receipt.committed_rounds, + "RBC-DAG frontier application commit watermarks must match its atomic receipt" + ); + commit.sub_dag + }) + .unwrap_or_default(); + RbcDagFrontierRecoveryCursorV1 { + receipt, + application_references, + } + }); + // Use genesis blocks cached in DagState (already inserted into DAG on // clean start by DagState::open()). Threshold clock is also initialized // inside DagState::open(). @@ -204,6 +337,7 @@ impl Core { committer, encoder, rbc_dag_application_production: false, + latest_rbc_dag_frontier_cursor, }; if !unprocessed_blocks.is_empty() { @@ -1562,29 +1696,128 @@ impl Core { self.flush_pending_clean_refs(); } - /// Persist an M7 frontier delta without feeding the obsolete Starfish - /// clean-DAG commit/proposal watermarks back into block production. - pub fn handle_rbc_dag_committed_delta(&mut self, committed: Vec) { + /// Atomically persist one authoritative RBC-DAG frontier and its optional + /// application commit. Classification against the in-memory durable + /// cursor happens before application materialization or observer effects. + pub(crate) fn handle_rbc_dag_committed_delta( + &mut self, + output_sequence: RoundNumber, + anchor: ConsensusVertexReference, + applications: &[BlockReference], + ) -> Result { let _timer = self .metrics .utilization_timer .utilization_timer("Core::handle_rbc_dag_committed_delta"); - let mut commit_data = Vec::with_capacity(committed.len()); - for commit in &committed { - self.dag_state.update_last_committed_rounds(commit); - commit_data.push(CommitData::new( - commit, - self.dag_state.last_committed_rounds(), - )); + + if let Some(current) = &self.latest_rbc_dag_frontier_cursor { + if output_sequence < current.receipt.output_sequence { + return Err(RbcDagFrontierApplyError::StaleSequence { + current_sequence: current.receipt.output_sequence, + actual_sequence: output_sequence, + }); + } + if output_sequence == current.receipt.output_sequence { + if anchor.carrier() == current.receipt.carrier_anchor { + if applications == current.application_references { + return Ok(RbcDagFrontierApplyOutcome::ExactReplay); + } + return Err(RbcDagFrontierApplyError::ConflictingApplications { + output_sequence, + anchor: anchor.carrier(), + expected: current.application_references.clone(), + actual: applications.to_vec(), + }); + } + return Err(RbcDagFrontierApplyError::ConflictingAnchor { + output_sequence, + expected: current.receipt.carrier_anchor, + actual: anchor.carrier(), + }); + } + if anchor.carrier() == current.receipt.carrier_anchor { + return Err(RbcDagFrontierApplyError::ReusedAnchor { + anchor: anchor.carrier(), + previous_sequence: current.receipt.output_sequence, + actual_sequence: output_sequence, + }); + } + let expected_sequence = current.receipt.output_sequence.checked_add(1).ok_or( + RbcDagFrontierApplyError::SequenceGap { + expected_sequence: RoundNumber::MAX, + actual_sequence: output_sequence, + }, + )?; + if output_sequence != expected_sequence { + return Err(RbcDagFrontierApplyError::SequenceGap { + expected_sequence, + actual_sequence: output_sequence, + }); + } + } else if output_sequence != 1 { + return Err(RbcDagFrontierApplyError::SequenceGap { + expected_sequence: 1, + actual_sequence: output_sequence, + }); } + + let blocks = applications + .iter() + .map(|reference| { + let block = self + .dag_state + .get_storage_block(*reference) + .ok_or(RbcDagFrontierApplyError::MissingApplication(*reference))?; + if !self.dag_state.is_data_available(reference) { + return Err(RbcDagFrontierApplyError::UnavailableApplication(*reference)); + } + Ok(block) + }) + .collect::, RbcDagFrontierApplyError>>()?; + let committed = CommittedSubDag::new(anchor.carrier(), blocks); + self.dag_state.update_last_committed_rounds(&committed); + let committed_rounds = self.dag_state.last_committed_rounds(); + let receipt = RbcDagFrontierReceipt { + carrier_anchor: anchor.carrier(), + output_sequence, + committed_rounds: committed_rounds.clone(), + }; + // A control-only frontier has no application CommitData, but the + // receipt still advances atomically through the same storage API. + let commit_data = (!applications.is_empty()) + .then(|| CommitData::new(&committed, committed_rounds)) + .into_iter() + .collect(); let store_start = std::time::Instant::now(); self.store - .store_commits(commit_data) + .store_commits_with_rbc_dag_receipt(commit_data, receipt.clone()) .expect("Store RBC-DAG frontier commits should not fail"); self.metrics .store_commits_latency_us .inc_by(store_start.elapsed().as_micros() as u64); self.metrics.store_commits_count.inc(); + self.latest_rbc_dag_frontier_cursor = Some(RbcDagFrontierRecoveryCursorV1 { + receipt, + application_references: applications.to_vec(), + }); + Ok(RbcDagFrontierApplyOutcome::Applied(vec![committed])) + } + + #[cfg(test)] + pub(crate) fn latest_rbc_dag_frontier_receipt(&self) -> Option { + self.latest_rbc_dag_frontier_cursor + .as_ref() + .map(|cursor| cursor.receipt.clone()) + } + + /// Return the exact runtime recovery cursor before Core is moved into its + /// dispatcher. The durable receipt intentionally remains compact; exact + /// application references are reconstructed from the atomic CommitData + /// stored under the carrier anchor. + pub(crate) fn rbc_dag_frontier_recovery_cursor( + &self, + ) -> Option { + self.latest_rbc_dag_frontier_cursor.clone() } pub(crate) fn enable_rbc_dag_application_production(&mut self) { @@ -1597,16 +1830,41 @@ impl Core { pub fn write_commits(&mut self, _commits: &[CommitData]) {} - pub fn take_recovered_committed(&mut self) -> (AHashSet, usize) { - let committed_blocks = self + pub fn take_recovered_committed( + &mut self, + rbc_dag_frontier_authority: bool, + ) -> (AHashSet, usize) { + let legacy_committed_blocks = self .recovered_committed_blocks .take() .expect("take_recovered_committed called twice"); - let committed_leaders_count = self + let legacy_committed_leaders_count = self .recovered_committed_leaders_count .take() .expect("take_recovered_committed called twice"); - (committed_blocks, committed_leaders_count) + if !rbc_dag_frontier_authority { + return (legacy_committed_blocks, legacy_committed_leaders_count); + } + + assert!( + self.dag_state.consensus_protocol.is_starfish_rbc(), + "embedded RBC-DAG observer recovery requires the Starfish-RBC protocol" + ); + // Carrier-keyed CommitData is intentionally not discoverable through + // DagState's Core-block scan, and the latest application CommitData + // contains only one frontier delta. Exactly-once is owned by the + // durable receipt plus the authoritative WAL; the observer needs only + // its monotone output count in this mode and never runs the legacy + // Linearizer. + let committed_frontier_count = self + .latest_rbc_dag_frontier_cursor + .as_ref() + .map(|cursor| { + usize::try_from(cursor.receipt.output_sequence) + .expect("RBC-DAG output sequence must fit usize") + }) + .unwrap_or_default(); + (AHashSet::new(), committed_frontier_count) } pub fn dag_state(&self) -> &DagState { @@ -1666,10 +1924,13 @@ mod tests { bls_certificate_aggregator::CertificateEvent, config::{DisseminationMode, NodePrivateConfig, StorageBackend}, crypto::{self, BlsSigner, Signer}, - dag_state::{DagState, DataSource}, + dag_state::{CommitData, DagState, DataSource}, data::Data, metrics::Metrics, - types::{AuthoritySet, BlockReference, BlsAggregateCertificate, VerifiedBlock}, + types::{ + AuthoritySet, BlockReference, BlsAggregateCertificate, Transaction, TransactionData, + VerifiedBlock, + }, }; struct NoopBlockHandler; @@ -1770,6 +2031,14 @@ mod tests { Data::new(block) } + fn rbc_application_is_materialized( + core: &Core, + reference: BlockReference, + ) -> bool { + core.dag_state().get_storage_block(reference).is_some() + && core.dag_state().is_data_available(&reference) + } + fn make_test_round_certificate( bls_signers: &[BlsSigner], round: RoundNumber, @@ -1970,6 +2239,441 @@ mod tests { ); } + #[test] + fn rbc_dag_control_frontier_cursor_reopens_and_binds_exact_empty_output() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let open = || { + DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ) + }; + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + NodePrivateConfig::new_for_tests(authority), + metrics.clone(), + open(), + None, + ); + let anchor = ConsensusVertexReference::new(BlockReference::new_test(2, 20), 7); + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, anchor, &[]), + Ok(RbcDagFrontierApplyOutcome::Applied(_)) + )); + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, anchor, &[]), + Ok(RbcDagFrontierApplyOutcome::ExactReplay) + )); + let unexpected = BlockReference::new_test(1, 3); + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, anchor, &[unexpected]), + Err(RbcDagFrontierApplyError::ConflictingApplications { + output_sequence: 1, + expected, + actual, + .. + }) if expected.is_empty() && actual == vec![unexpected] + )); + drop(core); + + let (mut reopened, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + NodePrivateConfig::new_for_tests(authority), + metrics.clone(), + open(), + None, + ); + let cursor = reopened + .rbc_dag_frontier_recovery_cursor() + .expect("control-only receipt must reopen as an exact cursor"); + assert_eq!(cursor.receipt.carrier_anchor, anchor.carrier()); + assert_eq!(cursor.receipt.output_sequence, 1); + assert!(cursor.application_references.is_empty()); + assert!(matches!( + reopened.handle_rbc_dag_committed_delta(1, anchor, &[]), + Ok(RbcDagFrontierApplyOutcome::ExactReplay) + )); + } + + #[test] + fn rbc_dag_application_frontier_cursor_reconstructs_exact_references() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let open = || { + DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ) + }; + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + NodePrivateConfig::new_for_tests(authority), + metrics.clone(), + open(), + None, + ); + let application = make_starfish_rbc_round_1_block(&committee, 1); + let application_reference = *application.reference(); + core.add_blocks(vec![(application, None)], DataSource::BlockBundleStreaming); + assert!(core.dag_state().is_data_available(&application_reference)); + let anchor = ConsensusVertexReference::new(BlockReference::new_test(3, 30), 9); + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, anchor, &[application_reference]), + Ok(RbcDagFrontierApplyOutcome::Applied(_)) + )); + drop(core); + + let (mut reopened, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + NodePrivateConfig::new_for_tests(authority), + metrics.clone(), + open(), + None, + ); + let cursor = reopened + .rbc_dag_frontier_recovery_cursor() + .expect("application frontier receipt must reopen with exact references"); + assert_eq!(cursor.application_references, vec![application_reference]); + assert!(matches!( + reopened.handle_rbc_dag_committed_delta(1, anchor, &[application_reference]), + Ok(RbcDagFrontierApplyOutcome::ExactReplay) + )); + assert!(matches!( + reopened.handle_rbc_dag_committed_delta(1, anchor, &[]), + Err(RbcDagFrontierApplyError::ConflictingApplications { .. }) + )); + } + + #[test] + fn rbc_dag_frontier_sequence_accepts_regressing_anchor_round_and_rejects_gaps() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee, + NodePrivateConfig::new_for_tests(authority), + metrics, + recovered, + None, + ); + let later_certifier = ConsensusVertexReference::new(BlockReference::new_test(2, 50), 8); + let older_leader = ConsensusVertexReference::new(BlockReference::new_test(1, 40), 3); + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, later_certifier, &[]), + Ok(RbcDagFrontierApplyOutcome::Applied(_)) + )); + assert!(matches!( + core.handle_rbc_dag_committed_delta(3, older_leader, &[]), + Err(RbcDagFrontierApplyError::SequenceGap { + expected_sequence: 2, + actual_sequence: 3 + }) + )); + assert!(matches!( + core.handle_rbc_dag_committed_delta(2, older_leader, &[]), + Ok(RbcDagFrontierApplyOutcome::Applied(_)) + )); + let receipt = core.latest_rbc_dag_frontier_receipt().unwrap(); + assert_eq!(receipt.output_sequence, 2); + assert_eq!(receipt.carrier_anchor, older_leader.carrier()); + let (legacy_refs, observer_count) = core.take_recovered_committed(true); + assert!(legacy_refs.is_empty()); + assert_eq!(observer_count, 2); + } + + #[test] + #[should_panic(expected = "a present RBC-DAG frontier application commit must not be empty")] + fn rbc_dag_frontier_reopen_rejects_present_empty_commit_data() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let anchor = BlockReference::new_test(2, 7); + let receipt = RbcDagFrontierReceipt { + carrier_anchor: anchor, + output_sequence: 1, + committed_rounds: vec![0; committee.len()], + }; + recovered + .store + .store_commits_with_rbc_dag_receipt(Vec::new(), receipt.clone()) + .unwrap(); + recovered + .store + .store_commits(vec![CommitData { + leader: anchor, + sub_dag: Vec::new(), + committed_rounds: receipt.committed_rounds, + }]) + .unwrap(); + + let _ = Core::open( + NoopBlockHandler, + authority, + committee, + NodePrivateConfig::new_for_tests(authority), + metrics, + recovered, + None, + ); + } + + #[test] + fn rbc_dag_frontier_rejects_missing_or_unavailable_applications_without_advancing_receipt() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let private_config = NodePrivateConfig::new_for_tests(authority); + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + private_config, + metrics, + recovered, + None, + ); + let anchor = ConsensusVertexReference::new(BlockReference::new_test(authority, 1), 1); + let missing = BlockReference::new_test(1, 7); + + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, anchor, &[missing]), + Err(RbcDagFrontierApplyError::MissingApplication(reference)) if reference == missing + )); + assert!(core.latest_rbc_dag_frontier_receipt().is_none()); + + let transactions = vec![BaseTransaction::Share(Transaction::new(vec![9; 64]))]; + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded = encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + let mut unavailable = VerifiedBlock::new_starfish_rbc( + 1, + 1, + committee + .authorities() + .map(|parent| BlockReference::new_test(parent, 0)) + .collect(), + Vec::new(), + 1, + Vec::new(), + Some(encoded), + ); + unavailable.preserialize(); + let unavailable = Data::new(unavailable); + let unavailable_reference = *unavailable.reference(); + let (processed, missing_parents, processed_references, _) = + core.add_headers(vec![unavailable], DataSource::BlockBundleStreamingHeader); + assert!(processed); + assert!(missing_parents.is_empty()); + assert!(processed_references.contains(&unavailable_reference)); + assert!(!core.dag_state().is_data_available(&unavailable_reference)); + + assert!(matches!( + core.handle_rbc_dag_committed_delta(1, anchor, &[unavailable_reference]), + Err(RbcDagFrontierApplyError::UnavailableApplication(reference)) + if reference == unavailable_reference + )); + assert!(core.latest_rbc_dag_frontier_receipt().is_none()); + } + + #[test] + fn buffered_payload_materializes_only_after_missing_header_parents_arrive() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let private_config = NodePrivateConfig::new_for_tests(authority); + let (mut core, _) = Core::open( + NoopBlockHandler, + authority, + committee.clone(), + private_config, + metrics, + recovered, + None, + ); + + let parents = [1, 2, 3] + .into_iter() + .map(|peer| make_starfish_rbc_round_1_block(&committee, peer)) + .collect::>(); + let parent_references = parents + .iter() + .map(|parent| *parent.reference()) + .collect::>(); + let transactions = vec![BaseTransaction::Share(Transaction::new(vec![7; 64]))]; + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded = encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + let mut child = VerifiedBlock::new_starfish_rbc( + 1, + 2, + parent_references.clone(), + Vec::new(), + 2, + Vec::new(), + Some(encoded.clone()), + ); + child.preserialize(); + let child = Data::new(child); + let child_reference = *child.reference(); + + let mut transaction_data = TransactionData::new(transactions); + transaction_data.preserialize(); + let (commitment, proof) = + crypto::TransactionsCommitment::new_from_encoded_transactions(&encoded, 0); + let mut shard_data = ProvableShard::new(encoded[0].clone(), 0, proof, commitment); + shard_data.preserialize(); + + // Payload-first arrival is buffered because no dependency-closed + // DagState block exists. It must not be treated as availability. + core.add_transaction_data( + vec![ReconstructedTransactionData { + block_reference: child_reference, + transaction_data, + shard_data, + }], + DataSource::StarfishRbcPayload, + ); + assert_eq!(core.pending_reconstructed_data.len(), 1); + assert!(!rbc_application_is_materialized(&core, child_reference)); + + let (processed, missing, processed_references, _) = + core.add_headers(vec![child], DataSource::BlockBundleStreamingHeader); + assert!(!processed); + assert!(!missing.is_empty()); + assert!(!processed_references.contains(&child_reference)); + assert!(!rbc_application_is_materialized(&core, child_reference)); + + // Adding the parents activates the pending child and atomically + // attaches the buffered payload. HeaderStaged must use this returned + // processed-reference set to emit the delayed availability signal. + let (processed, missing, processed_references, _) = + core.add_headers(parents, DataSource::BlockBundleStreamingHeader); + assert!(processed); + assert!(missing.is_empty()); + assert!(processed_references.contains(&child_reference)); + assert!(core.pending_reconstructed_data.is_empty()); + assert!(rbc_application_is_materialized(&core, child_reference)); + } + #[test] fn mysticeti_bls_non_leader_can_build_round_2_with_prev_leader_parent() { let authority = 0; diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 56f2fcad..b7ba6a73 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -10,10 +10,11 @@ use tokio::sync::{mpsc, oneshot}; use crate::{ block_handler::BlockHandler, bls_certificate_aggregator::CertificateEvent, + core::RbcDagFrontierApplyError, dag_state::DataSource, data::Data, metrics::{Metrics, UtilizationTimerExt}, - starfish_rbc::PinnedRbcHeader, + starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, starfish_rbc_dag_shadow::CommittedFrontierDeltaV1, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ @@ -53,6 +54,14 @@ enum CoreThreadCommand { DataSource, oneshot::Sender<()>, ), + /// Header authority established by the single-owner RBC-DAG carrier + /// actor. This typed command cannot be forged through `BlockBatch.source`. + AddAuthorizedRbcDagHeader( + RbcCanonicalHeader, + oneshot::Sender<(AHashSet, Vec)>, + ), + /// Payload verified against an RBC-DAG-authorized canonical header. + AddAuthorizedRbcDagPayload(ReconstructedTransactionData, oneshot::Sender<()>), MissingParentReferences(oneshot::Sender>), ForceNewBlock(RoundNumber, oneshot::Sender<()>), /// Attempt block creation with relaxed readiness checks (StarfishSpeed soft @@ -73,7 +82,16 @@ enum CoreThreadCommand { /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. - ApplyStarfishRbcDagFrontier(CommittedFrontierDeltaV1, oneshot::Sender<()>), + ApplyStarfishRbcDagFrontier( + CommittedFrontierDeltaV1, + oneshot::Sender>, + ), + /// Release production only after the ordered recovery bridge processes + /// the service's final startup `Ready` event. + ActivateStarfishRbcDagAuthority(oneshot::Sender<()>), + /// Release the one-outstanding application-production gate after the + /// exact header is durably assigned to a local carrier. + ApplyStarfishRbcDagApplicationAssigned(BlockReference, oneshot::Sender<()>), /// Store a Sailfish++ timeout certificate in DagState. ApplyTimeoutCert(SailfishTimeoutCert, oneshot::Sender<()>), /// Store a Sailfish++ no-vote certificate in DagState. @@ -98,9 +116,13 @@ impl Syncer { + pub fn stop(self) -> thread::Result> { drop(self.sender); - self.join_handle.join().unwrap() + self.join_handle.join() + } + + pub fn is_finished(&self) -> bool { + self.join_handle.is_finished() } pub async fn add_blocks( @@ -140,6 +162,23 @@ impl (AHashSet, Vec) { + let (sender, receiver) = oneshot::channel(); + self.send(CoreThreadCommand::AddAuthorizedRbcDagHeader(header, sender)) + .await; + receiver.await.expect("core thread is not expected to stop") + } + + pub(crate) async fn add_authorized_rbc_dag_payload(&self, item: ReconstructedTransactionData) { + let (sender, receiver) = oneshot::channel(); + self.send(CoreThreadCommand::AddAuthorizedRbcDagPayload(item, sender)) + .await; + receiver.await.expect("core thread is not expected to stop") + } + pub async fn missing_parent_references(&self) -> Vec { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::MissingParentReferences(sender)) @@ -147,6 +186,25 @@ impl bool { + let (sender, receiver) = oneshot::channel(); + self.metrics.core_lock_enqueued.inc(); + self.metrics.core_queue_length.inc(); + if self + .sender + .send(CoreThreadCommand::MissingParentReferences(sender)) + .await + .is_err() + { + self.metrics.core_queue_length.dec(); + return false; + } + receiver.await.is_ok() + } + pub async fn force_commit(&self) { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ForceCommit(sender)).await; @@ -214,12 +272,34 @@ impl Result { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ApplyStarfishRbcDagFrontier( delta, sender, )) .await; + receiver.await.expect("core thread is not expected to stop") + } + + pub(crate) async fn activate_starfish_rbc_dag_authority(&self) { + let (sender, receiver) = oneshot::channel(); + self.send(CoreThreadCommand::ActivateStarfishRbcDagAuthority(sender)) + .await; + receiver.await.expect("core thread is not expected to stop"); + } + + pub(crate) async fn apply_starfish_rbc_dag_application_assigned( + &self, + reference: BlockReference, + ) { + let (sender, receiver) = oneshot::channel(); + self.send(CoreThreadCommand::ApplyStarfishRbcDagApplicationAssigned( + reference, sender, + )) + .await; receiver.await.expect("core thread is not expected to stop"); } @@ -307,6 +387,22 @@ impl CoreThread { self.syncer.add_transaction_data(items, source); sender.send(()).ok(); } + CoreThreadCommand::AddAuthorizedRbcDagHeader(header, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["add_authorized_rbc_dag_header"]) + .inc(); + let result = self.syncer.add_authorized_rbc_dag_header(header); + sender.send(result).ok(); + } + CoreThreadCommand::AddAuthorizedRbcDagPayload(item, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["add_authorized_rbc_dag_payload"]) + .inc(); + self.syncer.add_authorized_rbc_dag_payload(item); + sender.send(()).ok(); + } CoreThreadCommand::MissingParentReferences(sender) => { metrics .core_thread_tasks_total @@ -419,7 +515,24 @@ impl CoreThread { .core_thread_tasks_total .with_label_values(&["apply_starfish_rbc_dag_frontier"]) .inc(); - self.syncer.apply_starfish_rbc_dag_frontier(delta); + let result = self.syncer.apply_starfish_rbc_dag_frontier(delta); + sender.send(result).ok(); + } + CoreThreadCommand::ActivateStarfishRbcDagAuthority(sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["activate_starfish_rbc_dag_authority"]) + .inc(); + self.syncer.activate_starfish_rbc_dag_authority(); + sender.send(()).ok(); + } + CoreThreadCommand::ApplyStarfishRbcDagApplicationAssigned(reference, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_dag_application_assigned"]) + .inc(); + self.syncer + .apply_starfish_rbc_dag_application_assigned(reference); sender.send(()).ok(); } CoreThreadCommand::ApplyTimeoutCert(cert, sender) => { @@ -486,14 +599,7 @@ mod tests { Vec::new() } - fn handle_rbc_dag_commit( - &mut self, - _dag_state: &DagState, - _anchor: BlockReference, - _applications: &[BlockReference], - ) -> Vec { - Vec::new() - } + fn handle_rbc_dag_commit(&mut self, _committed: &[CommittedSubDag]) {} fn recover_committed( &mut self, @@ -604,6 +710,6 @@ mod tests { let refs = dispatcher.missing_parent_references().await; assert_eq!(refs, vec![missing_earlier, missing_later]); - dispatcher.stop(); + assert!(dispatcher.stop().is_ok()); } } diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 8e008bb0..a659b2da 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -84,6 +84,14 @@ pub enum DataSource { RoundGapResponse, /// Transaction data co-carried by the direct Starfish-RBC INIT. StarfishRbcPayload, + /// Header authorized by an authenticated or delivered RBC-DAG carrier. + /// This is an internal provenance label; authorization is carried by the + /// dedicated core-thread command, never by the peer-controlled wire enum. + StarfishRbcDagAuthorizedHeader, + /// Payload verified against an RBC-DAG-authorized header commitment. + /// Like the header label, this is accepted only through the dedicated + /// core-thread command. + StarfishRbcDagAuthorizedPayload, } impl DataSource { @@ -99,6 +107,8 @@ impl DataSource { Self::UnprovableCertificateResponse => "unprovable_certificate_response", Self::RoundGapResponse => "round_gap_response", Self::StarfishRbcPayload => "starfish_rbc_payload", + Self::StarfishRbcDagAuthorizedHeader => "starfish_rbc_dag_authorized_header", + Self::StarfishRbcDagAuthorizedPayload => "starfish_rbc_dag_authorized_payload", } } } @@ -2534,6 +2544,33 @@ impl DagState { self.dag_state_inner.read().last_committed_rounds.clone() } + /// Restore the exact durable RBC-DAG application watermark before the + /// authoritative carrier actor is opened. The receipt is the only commit + /// record for a control-only frontier, so ordinary application-commit + /// recovery cannot reconstruct this vector on its own. + pub(crate) fn restore_rbc_dag_committed_rounds(&self, committed_rounds: &[RoundNumber]) { + let mut inner = self.dag_state_inner.write(); + assert_eq!( + committed_rounds.len(), + inner.committee_size, + "RBC-DAG frontier receipt watermark length must match the committee" + ); + for (authority, (recovered, durable)) in inner + .last_committed_rounds + .iter() + .zip(committed_rounds) + .enumerate() + { + assert!( + durable >= recovered, + "RBC-DAG frontier receipt regresses recovered authority {authority}: durable {durable}, recovered {recovered}" + ); + } + inner + .last_committed_rounds + .clone_from_slice(committed_rounds); + } + pub fn cleanup(&self) { let _timer = self.metrics.dag_state_cleanup_util.utilization_timer(); diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index aabc76f7..f118bc4a 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -7,7 +7,7 @@ use std::{ ops::AddAssign, sync::{ Arc, - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, }, time::Duration, }; @@ -26,6 +26,9 @@ use crate::{ committee::Committee, data::{IN_MEMORY_BLOCKS, IN_MEMORY_BLOCKS_BYTES}, runtime, + starfish_rbc_dag::model::{ + EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, + }, stat::{DivUsize, HistogramSender, PreciseHistogram, histogram}, types::{AuthorityIndex, format_authority_index}, }; @@ -33,6 +36,43 @@ use crate::{ /// Metrics collected by the benchmark. pub const BENCHMARK_DURATION: &str = "benchmark_duration"; +/// One absolute submission window shared by every local-benchmark generator. +/// The common epoch removes sequential-start and polling skew from offered +/// load, cutoff counters, and latency samples. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BenchmarkTransactionWindow { + pub start: Instant, + pub end: Instant, +} + +impl BenchmarkTransactionWindow { + pub fn new(start: Instant, end: Instant) -> Option { + (start < end).then_some(Self { start, end }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +pub enum BenchmarkGeneratorState { + Disabled = 0, + Waiting = 1, + Active = 2, + Finished = 3, + Failed = 4, +} + +impl BenchmarkGeneratorState { + pub fn from_u8(value: u8) -> Self { + match value { + 1 => Self::Waiting, + 2 => Self::Active, + 3 => Self::Finished, + 4 => Self::Failed, + _ => Self::Disabled, + } + } +} + pub const TRANSACTION_CERTIFIED_LATENCY: &str = "transaction_certified_latency"; pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; @@ -49,7 +89,25 @@ pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; /// while still detecting an actor that is no longer draining work. pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG: i64 = 4; pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR: i64 = 16; -pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR: i64 = 2; +/// Per-remote-author capacity of authenticated future slots that are inside +/// the executable retention window but outside its immediate admission +/// window. This is an asynchronous safety bound, not a healthy-tail target. +pub const STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_CAPACITY_PER_REMOTE: i64 = + EXECUTABLE_MODEL_BUFFER_WINDOW_V1 as i64 - EXECUTABLE_MODEL_ADMISSION_WINDOW_V1 as i64; +/// With final honest round skew bounded by four and two future rounds admitted, +/// at most two slots per remote author remain buffered in a settled run. +pub const STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_SETTLED_PER_REMOTE: i64 = + STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG - EXECUTABLE_MODEL_ADMISSION_WINDOW_V1 as i64; + +pub const fn starfish_rbc_dag_autonomous_buffered_capacity_bound(committee_size: i64) -> i64 { + STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_CAPACITY_PER_REMOTE + .saturating_mul(committee_size.saturating_sub(1)) +} + +pub const fn starfish_rbc_dag_autonomous_buffered_settled_bound(committee_size: i64) -> i64 { + STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_SETTLED_PER_REMOTE + .saturating_mul(committee_size.saturating_sub(1)) +} const LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES: &[&str] = &[ "subscribe_broadcast", @@ -75,6 +133,34 @@ const LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES: &[&str] = &[ "rbc_dag_shadow_carrier_response", "rbc_dag_shadow_carrier_sync_request", "rbc_dag_shadow_carrier_sync_response", + "rbc_dag_application_payload_request", + "rbc_dag_application_payload_response", +]; + +pub(crate) const RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT: &str = "creation_to_assignment"; +pub(crate) const RBC_DAG_LATENCY_CREATION_TO_DELIVERY: &str = "creation_to_delivery"; +pub(crate) const RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED: &str = + "creation_to_frontier_generated"; +pub(crate) const RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED: &str = + "creation_to_frontier_applied"; + +const RBC_DAG_PIPELINE_LATENCY_STAGES: &[&str] = &[ + RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, + RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, + RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, +]; +pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD: &str = "physical_forward"; +pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD: &str = "physical_backward"; +const RBC_DAG_COMMIT_DISTANCE_KINDS: &[&str] = &[ + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, +]; +const RBC_DAG_PROJECTION_HOL_STATES: &[&str] = &[ + "insufficient_lookahead", + "direct_evidence_pending", + "awaiting_indirect_anchor", + "ready", ]; #[derive(Clone)] @@ -84,6 +170,10 @@ pub struct Metrics { pub leader_timeout_total: IntCounter, pub proposal_wait_time_total_us: IntCounter, pub sequenced_transactions_total: IntCounter, + /// Transactions committed before the coordinated benchmark's exact + /// monotonic cutoff. The ordinary sequenced counter remains open through + /// the bounded drain to measure eventual active-window throughput. + pub sequenced_transactions_cutoff_total: IntCounter, pub sequenced_transactions_bytes: IntCounter, pub sailfish_rbc_fast_total: IntCounter, pub sailfish_rbc_slow_total: IntCounter, @@ -200,6 +290,40 @@ pub struct Metrics { pub starfish_rbc_dag_shadow_buffered_authenticated: IntGauge, pub starfish_rbc_dag_projected_vertices_total: IntCounter, pub starfish_rbc_dag_projection_decisions_total: IntCounterVec, + /// Active-window application latency decomposed by one of four fixed + /// pipeline stages. Keeping sum/count/max avoids per-block labels and the + /// cost of a high-volume histogram on the carrier actor's hot path. + pub starfish_rbc_dag_pipeline_latency_ns_total: IntCounterVec, + pub starfish_rbc_dag_pipeline_latency_samples_total: IntCounterVec, + pub starfish_rbc_dag_pipeline_latency_ns_max: IntGaugeVec, + /// Diagnostic-only round distances for first-committed applications. + /// Physical deltas use separate forward/backward magnitude labels so + /// cross-author clock skew is not hidden by unsigned saturation. + pub starfish_rbc_dag_commit_distance_rounds_total: IntCounterVec, + pub starfish_rbc_dag_commit_distance_samples_total: IntCounterVec, + pub starfish_rbc_dag_commit_distance_rounds_max: IntGaugeVec, + /// Current and process-high-water queue depths, with the bounded labels + /// `local` and `projection`. + pub starfish_rbc_dag_pipeline_queue_depth: IntGaugeVec, + pub starfish_rbc_dag_pipeline_queue_depth_max: IntGaugeVec, + pub starfish_rbc_dag_highest_projected_consensus_round: IntGauge, + pub starfish_rbc_dag_next_undecided_consensus_round: IntGauge, + pub starfish_rbc_dag_next_undecided_projected_stake: IntGauge, + pub starfish_rbc_dag_last_committed_consensus_round: IntGauge, + /// One-hot current projection head-of-line state. The label vocabulary is + /// fixed in `set_starfish_rbc_dag_pipeline_state`. + pub starfish_rbc_dag_projection_hol_state: IntGaugeVec, + /// Frontier lifecycle counters plus the current/high-water number created + /// by the synchronous carrier actor but not yet applied by the core + /// dispatcher. + pub starfish_rbc_dag_frontier_events_total: IntCounterVec, + pub starfish_rbc_dag_frontiers_inflight: IntGauge, + pub starfish_rbc_dag_frontiers_inflight_max: IntGauge, + /// Sequenced-transaction total observed by the frontier bridge only after + /// it records the corresponding final application-latency samples. The + /// local benchmark uses this release/acquire acknowledgement before + /// closing its post-cutoff transaction-observation gate. + starfish_rbc_dag_frontier_applied_sequenced_transactions: Arc, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -235,11 +359,22 @@ pub struct Metrics { /// True iff the validator is inside the active transaction-submission /// window. Outside this window — during the warmup before the first - /// transaction is generated, and after the generator stops — every - /// rate-relevant metric update (latency observations, sequenced / - /// committed counters, the `benchmark_duration` clock) is skipped, so - /// reported TPS / BPS / p50 latency reflect only the steady-state window. + /// transaction is generated, and after the generator stops — protocol + /// throughput/latency metrics and the `benchmark_duration` clock are + /// skipped. Transaction commits use `transaction_metrics_active` so the + /// offered window can be followed through a bounded drain. pub metrics_active: Arc, + /// Transaction and application-pipeline observations remain enabled + /// during the bounded post-window drain, while all ordinary + /// protocol/window metrics close exactly at the shared cutoff. This makes + /// active-window latency uncensored without charging drain traffic to + /// block/RBC throughput. + pub transaction_metrics_active: Arc, + /// Runtime-only coordinated generator lifecycle for local benchmarks. + pub benchmark_generator_state: Arc, + /// Common benchmark cutoff expressed in microseconds since this + /// validator's `validator_start`; zero outside coordinated benchmarks. + pub benchmark_transaction_cutoff_micros: Arc, /// Wall-clock instant the validator's metrics were first activated, in /// microseconds since `validator_start`. Used by the /// `benchmark_duration` Prometheus counter so its denominator counts @@ -279,6 +414,38 @@ pub struct AutonomousClockBenchmarkBaseline { carrier_round: i64, } +/// Immutable per-validator RBC-DAG state sampled at the common transaction +/// cutoff. The bounded post-window transaction drain may improve or worsen +/// live gauges, but it must never rewrite the verdict for the measured +/// interval. +#[derive(Clone, Copy, Debug, Default)] +pub struct AutonomousClockBenchmarkSnapshot { + accepted_heartbeats: u64, + accepted_application_carriers: u64, + delivered_carriers: u64, + delivered_applications: u64, + committed_frontiers: u64, + frontier_applications: u64, + projected_vertices: u64, + projection_decisions: u64, + wal_batches: u64, + wal_records: u64, + clock_valid: i64, + carrier_round: i64, + phase_backlog: i64, + admitted_authors: i64, + admitted_stake: i64, + buffered_authenticated: i64, + pending_recovery: i64, +} + +impl AutonomousClockBenchmarkSnapshot { + fn accepted_local_carriers(self) -> u64 { + self.accepted_heartbeats + .saturating_add(self.accepted_application_carriers) + } +} + /// Per-validator cumulative counters sampled at the exact start of a local /// benchmark's active transaction window. Rates subtract this snapshot so /// connection warmup and shadow-WAL replay are not charged to the protocol. @@ -291,6 +458,20 @@ pub struct LocalBenchmarkCounterBaseline { outbound_messages: Vec<(u64, u64)>, } +#[derive(Clone, Copy, Debug)] +pub struct LocalBenchmarkTransactionOutcome { + /// Exact successful sends across all honest local generators in the + /// shared active window. Byzantine generators are disabled by the local + /// harness, so this is the global set every honest validator must drain. + pub offered_transactions: u64, + /// Mean per-honest-validator commits observed at the common cutoff. + pub cutoff_committed_transactions: u64, + /// Mean per-honest-validator commits after the bounded drain. + pub eventual_committed_transactions: u64, + pub drain_elapsed: Duration, + pub drain_complete: bool, +} + #[derive(Debug, Eq, PartialEq)] struct AutonomousClockBenchmarkSummary { valid_nodes: usize, @@ -321,200 +502,134 @@ fn summarize_autonomous_clock_benchmark( metrics: &[Arc], committee_size: usize, baselines: Option<&[AutonomousClockBenchmarkBaseline]>, + cutoff_snapshots: Option<&[AutonomousClockBenchmarkSnapshot]>, embedded_rbc_authority: bool, ) -> AutonomousClockBenchmarkSummary { let committee_size = i64::try_from(committee_size).unwrap_or(i64::MAX); let maximum_phase_backlog_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR.saturating_mul(committee_size); let maximum_buffered_authenticated_bound = - STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR.saturating_mul(committee_size); + starfish_rbc_dag_autonomous_buffered_settled_bound(committee_size); + + // A supplied cutoff vector is authoritative. Missing entries fail closed + // to the all-zero default rather than falling back to mutable drain-time + // gauges and accidentally turning an invalid measured run into VALID. + let observations = metrics + .iter() + .enumerate() + .map(|(index, metrics)| match cutoff_snapshots { + Some(snapshots) => snapshots.get(index).copied().unwrap_or_default(), + None => metrics.autonomous_clock_benchmark_snapshot(), + }) + .collect::>(); - let valid_nodes = metrics + let valid_nodes = observations .iter() - .filter(|metrics| metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1) + .filter(|snapshot| snapshot.clock_valid == 1) .count(); - let progress_nodes = metrics + let progress_nodes = observations .iter() .enumerate() - .filter(|(index, metrics)| { + .filter(|(index, snapshot)| { let baseline = baselines .and_then(|baselines| baselines.get(*index)) .copied() .unwrap_or_default(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .get() - .saturating_add( - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["application_carrier", "accepted"]) - .get(), - ) - > baseline.accepted_local_carriers - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get() - > baseline.delivered_carriers + snapshot.accepted_local_carriers() > baseline.accepted_local_carriers + && snapshot.delivered_carriers > baseline.delivered_carriers && (!embedded_rbc_authority - || metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .get() - > baseline.delivered_applications - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get() - > baseline.committed_frontiers - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get() - > baseline.frontier_applications) - && metrics.starfish_rbc_dag_projected_vertices_total.get() - > baseline.projected_vertices - && metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get() - > baseline.projection_decisions - && metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total - .get() - > baseline.wal_batches - && metrics - .starfish_rbc_dag_shadow_wal_appended_records_total - .get() - > baseline.wal_records - && metrics.starfish_rbc_dag_shadow_carrier_round.get() > baseline.carrier_round + || snapshot.delivered_applications > baseline.delivered_applications + && snapshot.committed_frontiers > baseline.committed_frontiers + && snapshot.frontier_applications > baseline.frontier_applications) + && snapshot.projected_vertices > baseline.projected_vertices + && snapshot.projection_decisions > baseline.projection_decisions + && snapshot.wal_batches > baseline.wal_batches + && snapshot.wal_records > baseline.wal_records + && snapshot.carrier_round > baseline.carrier_round }) .count(); - let bounded_nodes = metrics + let bounded_nodes = observations .iter() - .filter(|metrics| { - let phase_backlog = metrics.starfish_rbc_dag_shadow_phase_backlog.get(); - let admitted_authors = metrics.starfish_rbc_dag_shadow_admitted_authors.get(); - let admitted_stake = metrics.starfish_rbc_dag_shadow_admitted_stake.get(); - let buffered = metrics.starfish_rbc_dag_shadow_buffered_authenticated.get(); - phase_backlog >= 0 - && phase_backlog <= maximum_phase_backlog_bound - && admitted_authors >= 0 - && admitted_authors <= committee_size - && admitted_stake >= 0 - && buffered >= 0 - && buffered <= maximum_buffered_authenticated_bound - && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 + .filter(|snapshot| { + snapshot.phase_backlog >= 0 + && snapshot.phase_backlog <= maximum_phase_backlog_bound + && snapshot.admitted_authors >= 0 + && snapshot.admitted_authors <= committee_size + && snapshot.admitted_stake >= 0 + && snapshot.buffered_authenticated >= 0 + && snapshot.buffered_authenticated <= maximum_buffered_authenticated_bound + && snapshot.pending_recovery == 0 }) .count(); - let accepted_heartbeats = metrics + let accepted_heartbeats = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .get() - }) + .map(|snapshot| snapshot.accepted_heartbeats) .sum(); - let delivered_carriers = metrics + let delivered_carriers = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get() - }) + .map(|snapshot| snapshot.delivered_carriers) .sum(); - let delivered_applications = metrics + let delivered_applications = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .get() - }) + .map(|snapshot| snapshot.delivered_applications) .sum(); - let committed_frontiers = metrics + let committed_frontiers = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get() - }) + .map(|snapshot| snapshot.committed_frontiers) .sum(); - let frontier_applications = metrics + let frontier_applications = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get() - }) + .map(|snapshot| snapshot.frontier_applications) .sum(); - let projected_vertices = metrics + let projected_vertices = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_projected_vertices_total.get()) + .map(|snapshot| snapshot.projected_vertices) .sum(); - let projection_decisions = metrics + let projection_decisions = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get() - }) + .map(|snapshot| snapshot.projection_decisions) .sum(); - let wal_batches = metrics + let wal_batches = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total - .get() - }) + .map(|snapshot| snapshot.wal_batches) .sum(); - let wal_records = metrics + let wal_records = observations .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_wal_appended_records_total - .get() - }) + .map(|snapshot| snapshot.wal_records) .sum(); - let pending_recovery = metrics + let pending_recovery = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_pending_recovery.get()) + .map(|snapshot| snapshot.pending_recovery) .sum(); - let minimum_round = metrics + let minimum_round = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) + .map(|snapshot| snapshot.carrier_round) .min() .unwrap_or_default(); - let maximum_round = metrics + let maximum_round = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) + .map(|snapshot| snapshot.carrier_round) .max() .unwrap_or_default(); - let maximum_phase_backlog = metrics + let maximum_phase_backlog = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_phase_backlog.get()) + .map(|snapshot| snapshot.phase_backlog) .max() .unwrap_or_default(); - let maximum_admitted_authors = metrics + let maximum_admitted_authors = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_authors.get()) + .map(|snapshot| snapshot.admitted_authors) .max() .unwrap_or_default(); - let maximum_admitted_stake = metrics + let maximum_admitted_stake = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_stake.get()) + .map(|snapshot| snapshot.admitted_stake) .max() .unwrap_or_default(); - let maximum_buffered_authenticated = metrics + let maximum_buffered_authenticated = observations .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_buffered_authenticated.get()) + .map(|snapshot| snapshot.buffered_authenticated) .max() .unwrap_or_default(); let round_lag = maximum_round.saturating_sub(minimum_round); @@ -563,18 +678,176 @@ pub struct VecHistogramReporter { gauge: IntGaugeVec, } +fn set_gauge_max(gauge: &IntGauge, value: i64) { + if value > gauge.get() { + gauge.set(value); + } +} + +fn format_rbc_dag_round_distance(total: u64, samples: u64, maximum: i64) -> String { + let average = if samples == 0 { + 0.0 + } else { + total as f64 / samples as f64 + }; + format!("{average:.2}/{maximum} rounds (n={samples})") +} + impl Metrics { + pub(crate) fn observe_starfish_rbc_dag_pipeline_latency_ns( + &self, + stage: &'static str, + total_ns: u64, + samples: u64, + max_ns: u64, + ) { + debug_assert!(RBC_DAG_PIPELINE_LATENCY_STAGES.contains(&stage)); + // These stages follow the finite set of applications offered during + // the common transaction window. Keep them open through the bounded + // drain so delivery/frontier latency is not right-censored at the + // submission cutoff. Protocol rates and round-distance observations + // remain scoped by `metrics_active` below. + if samples == 0 || !self.transaction_metrics_active.load(Ordering::Relaxed) { + return; + } + self.starfish_rbc_dag_pipeline_latency_ns_total + .with_label_values(&[stage]) + .inc_by(total_ns); + self.starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[stage]) + .inc_by(samples); + set_gauge_max( + &self + .starfish_rbc_dag_pipeline_latency_ns_max + .with_label_values(&[stage]), + i64::try_from(max_ns).unwrap_or(i64::MAX), + ); + } + + pub(crate) fn observe_starfish_rbc_dag_commit_round_distance( + &self, + kind: &'static str, + total_rounds: u64, + samples: u64, + max_rounds: u64, + ) { + debug_assert!(RBC_DAG_COMMIT_DISTANCE_KINDS.contains(&kind)); + if samples == 0 || !self.metrics_active.load(Ordering::Relaxed) { + return; + } + self.starfish_rbc_dag_commit_distance_rounds_total + .with_label_values(&[kind]) + .inc_by(total_rounds); + self.starfish_rbc_dag_commit_distance_samples_total + .with_label_values(&[kind]) + .inc_by(samples); + set_gauge_max( + &self + .starfish_rbc_dag_commit_distance_rounds_max + .with_label_values(&[kind]), + i64::try_from(max_rounds).unwrap_or(i64::MAX), + ); + } + + pub(crate) fn set_starfish_rbc_dag_pipeline_state( + &self, + pending_local: usize, + pending_projection: usize, + highest_projected_round: u32, + next_undecided_round: u32, + next_undecided_projected_stake: u64, + last_committed_round: u32, + hol_state: &'static str, + ) { + debug_assert!(RBC_DAG_PROJECTION_HOL_STATES.contains(&hol_state)); + for (queue, depth) in [("local", pending_local), ("projection", pending_projection)] { + let depth = i64::try_from(depth).unwrap_or(i64::MAX); + self.starfish_rbc_dag_pipeline_queue_depth + .with_label_values(&[queue]) + .set(depth); + set_gauge_max( + &self + .starfish_rbc_dag_pipeline_queue_depth_max + .with_label_values(&[queue]), + depth, + ); + } + self.starfish_rbc_dag_highest_projected_consensus_round + .set(i64::from(highest_projected_round)); + self.starfish_rbc_dag_next_undecided_consensus_round + .set(i64::from(next_undecided_round)); + self.starfish_rbc_dag_next_undecided_projected_stake + .set(i64::try_from(next_undecided_projected_stake).unwrap_or(i64::MAX)); + self.starfish_rbc_dag_last_committed_consensus_round + .set(i64::from(last_committed_round)); + for state in RBC_DAG_PROJECTION_HOL_STATES { + self.starfish_rbc_dag_projection_hol_state + .with_label_values(&[state]) + .set(i64::from(*state == hol_state)); + } + } + + pub(crate) fn starfish_rbc_dag_frontier_generated(&self) { + self.starfish_rbc_dag_frontier_events_total + .with_label_values(&["generated"]) + .inc(); + self.starfish_rbc_dag_frontiers_inflight.inc(); + set_gauge_max( + &self.starfish_rbc_dag_frontiers_inflight_max, + self.starfish_rbc_dag_frontiers_inflight.get(), + ); + } + + pub(crate) fn starfish_rbc_dag_frontier_applied(&self) { + self.starfish_rbc_dag_frontier_events_total + .with_label_values(&["applied"]) + .inc(); + if self.starfish_rbc_dag_frontiers_inflight.get() > 0 { + self.starfish_rbc_dag_frontiers_inflight.dec(); + } + // `RbcDagAppliedFrontierObservationV1::observe` calls this only after + // recording creation-to-frontier-applied latency. Publishing the + // sequenced count last gives the benchmark an ordered drain barrier, + // without confusing application samples (blocks) with transactions. + self.starfish_rbc_dag_frontier_applied_sequenced_transactions + .store(self.sequenced_transactions_total.get(), Ordering::Release); + } + + pub(crate) fn starfish_rbc_dag_frontier_ignored(&self) { + self.starfish_rbc_dag_frontier_events_total + .with_label_values(&["ignored"]) + .inc(); + if self.starfish_rbc_dag_frontiers_inflight.get() > 0 { + self.starfish_rbc_dag_frontiers_inflight.dec(); + } + } + pub fn autonomous_clock_benchmark_baseline(&self) -> AutonomousClockBenchmarkBaseline { + let snapshot = self.autonomous_clock_benchmark_snapshot(); AutonomousClockBenchmarkBaseline { - accepted_local_carriers: self + accepted_local_carriers: snapshot.accepted_local_carriers(), + delivered_carriers: snapshot.delivered_carriers, + delivered_applications: snapshot.delivered_applications, + committed_frontiers: snapshot.committed_frontiers, + frontier_applications: snapshot.frontier_applications, + projected_vertices: snapshot.projected_vertices, + projection_decisions: snapshot.projection_decisions, + wal_batches: snapshot.wal_batches, + wal_records: snapshot.wal_records, + carrier_round: snapshot.carrier_round, + } + } + + pub fn autonomous_clock_benchmark_snapshot(&self) -> AutonomousClockBenchmarkSnapshot { + AutonomousClockBenchmarkSnapshot { + accepted_heartbeats: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["heartbeat", "accepted"]) - .get() - .saturating_add( - self.starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["application_carrier", "accepted"]) - .get(), - ), + .get(), + accepted_application_carriers: self + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["application_carrier", "accepted"]) + .get(), delivered_carriers: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) @@ -602,10 +875,30 @@ impl Metrics { wal_records: self .starfish_rbc_dag_shadow_wal_appended_records_total .get(), + clock_valid: self.starfish_rbc_dag_shadow_clock_valid.get(), carrier_round: self.starfish_rbc_dag_shadow_carrier_round.get(), + phase_backlog: self.starfish_rbc_dag_shadow_phase_backlog.get(), + admitted_authors: self.starfish_rbc_dag_shadow_admitted_authors.get(), + admitted_stake: self.starfish_rbc_dag_shadow_admitted_stake.get(), + buffered_authenticated: self.starfish_rbc_dag_shadow_buffered_authenticated.get(), + pending_recovery: self.starfish_rbc_dag_shadow_pending_recovery.get(), } } + /// Number of offered applications whose authoritative RBC-DAG frontier + /// has been applied locally and whose final pipeline-latency sample has + /// therefore already been recorded. + pub fn starfish_rbc_dag_frontier_applied_latency_samples(&self) -> u64 { + self.starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED]) + .get() + } + + pub fn starfish_rbc_dag_frontier_applied_sequenced_transactions(&self) -> u64 { + self.starfish_rbc_dag_frontier_applied_sequenced_transactions + .load(Ordering::Acquire) + } + pub fn local_benchmark_counter_baseline(&self) -> LocalBenchmarkCounterBaseline { LocalBenchmarkCounterBaseline { sequenced_transactions: self.sequenced_transactions_total.get(), @@ -1070,6 +1363,122 @@ impl Metrics { registry, ) .unwrap(), + starfish_rbc_dag_pipeline_latency_ns_total: + register_int_counter_vec_with_registry!( + "starfish_rbc_dag_pipeline_latency_ns_total", + "Active-window application latency nanoseconds accumulated by bounded RBC-DAG pipeline stage", + &["stage"], + registry, + ) + .unwrap(), + starfish_rbc_dag_pipeline_latency_samples_total: + register_int_counter_vec_with_registry!( + "starfish_rbc_dag_pipeline_latency_samples_total", + "Active-window application latency sample count by bounded RBC-DAG pipeline stage", + &["stage"], + registry, + ) + .unwrap(), + starfish_rbc_dag_pipeline_latency_ns_max: register_int_gauge_vec_with_registry!( + "starfish_rbc_dag_pipeline_latency_ns_max", + "Maximum active-window application latency nanoseconds by bounded RBC-DAG pipeline stage", + &["stage"], + registry, + ) + .unwrap(), + starfish_rbc_dag_commit_distance_rounds_total: + register_int_counter_vec_with_registry!( + "starfish_rbc_dag_commit_distance_rounds_total", + "Active-window round-distance magnitude accumulated for first-committed RBC-DAG applications", + &["kind"], + registry, + ) + .unwrap(), + starfish_rbc_dag_commit_distance_samples_total: + register_int_counter_vec_with_registry!( + "starfish_rbc_dag_commit_distance_samples_total", + "Active-window first-committed RBC-DAG application sample count by round-distance kind", + &["kind"], + registry, + ) + .unwrap(), + starfish_rbc_dag_commit_distance_rounds_max: + register_int_gauge_vec_with_registry!( + "starfish_rbc_dag_commit_distance_rounds_max", + "Maximum active-window first-committed RBC-DAG application round-distance magnitude", + &["kind"], + registry, + ) + .unwrap(), + starfish_rbc_dag_pipeline_queue_depth: register_int_gauge_vec_with_registry!( + "starfish_rbc_dag_pipeline_queue_depth", + "Current RBC-DAG pipeline queue depth by bounded queue name", + &["queue"], + registry, + ) + .unwrap(), + starfish_rbc_dag_pipeline_queue_depth_max: register_int_gauge_vec_with_registry!( + "starfish_rbc_dag_pipeline_queue_depth_max", + "Process-high-water RBC-DAG pipeline queue depth by bounded queue name", + &["queue"], + registry, + ) + .unwrap(), + starfish_rbc_dag_highest_projected_consensus_round: + register_int_gauge_with_registry!( + "starfish_rbc_dag_highest_projected_consensus_round", + "Highest clean consensus round projected by this RBC-DAG runtime", + registry, + ) + .unwrap(), + starfish_rbc_dag_next_undecided_consensus_round: + register_int_gauge_with_registry!( + "starfish_rbc_dag_next_undecided_consensus_round", + "Oldest clean projected consensus round not yet decided", + registry, + ) + .unwrap(), + starfish_rbc_dag_next_undecided_projected_stake: + register_int_gauge_with_registry!( + "starfish_rbc_dag_next_undecided_projected_stake", + "Projected distinct-author stake at the oldest undecided consensus round", + registry, + ) + .unwrap(), + starfish_rbc_dag_last_committed_consensus_round: + register_int_gauge_with_registry!( + "starfish_rbc_dag_last_committed_consensus_round", + "Highest consensus round whose committed frontier was generated", + registry, + ) + .unwrap(), + starfish_rbc_dag_projection_hol_state: register_int_gauge_vec_with_registry!( + "starfish_rbc_dag_projection_hol_state", + "One-hot current certified-projection head-of-line state", + &["reason"], + registry, + ) + .unwrap(), + starfish_rbc_dag_frontier_events_total: register_int_counter_vec_with_registry!( + "starfish_rbc_dag_frontier_events_total", + "Committed RBC-DAG frontier lifecycle events by bounded stage", + &["stage"], + registry, + ) + .unwrap(), + starfish_rbc_dag_frontiers_inflight: register_int_gauge_with_registry!( + "starfish_rbc_dag_frontiers_inflight", + "Committed RBC-DAG frontiers generated but not yet applied or ignored", + registry, + ) + .unwrap(), + starfish_rbc_dag_frontiers_inflight_max: register_int_gauge_with_registry!( + "starfish_rbc_dag_frontiers_inflight_max", + "Process-high-water committed RBC-DAG frontiers awaiting event application", + registry, + ) + .unwrap(), + starfish_rbc_dag_frontier_applied_sequenced_transactions: Arc::new(AtomicU64::new(0)), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1169,6 +1578,12 @@ impl Metrics { registry, ) .unwrap(), + sequenced_transactions_cutoff_total: register_int_counter_with_registry!( + "sequenced_transactions_cutoff_total", + "Transactions sequenced before the coordinated benchmark cutoff", + registry, + ) + .unwrap(), sequenced_transactions_bytes: register_int_counter_with_registry!( "sequenced_transactions_bytes", "Total bytes of sequenced transactions", @@ -1515,6 +1930,11 @@ impl Metrics { // bound). The transaction generator overrides to false during // its warmup when the orchestrator sets a finite duration. metrics_active: Arc::new(AtomicBool::new(true)), + transaction_metrics_active: Arc::new(AtomicBool::new(true)), + benchmark_generator_state: Arc::new(AtomicU8::new( + BenchmarkGeneratorState::Disabled as u8, + )), + benchmark_transaction_cutoff_micros: Arc::new(AtomicU64::new(0)), active_start_micros: Arc::new(AtomicU64::new(0)), validator_start: tokio::time::Instant::now(), }; @@ -1531,7 +1951,10 @@ impl Metrics { starfish_rbc_dag_autonomous_clock_expected: bool, starfish_rbc_dag_embedded_rbc_authority_expected: bool, autonomous_clock_baselines: Option>, + autonomous_clock_cutoffs: Option>, counter_baselines: Option>, + counter_cutoffs: Option>, + transaction_outcome: Option, ) { let num_validators = metrics.len() as u64; @@ -1550,19 +1973,27 @@ impl Metrics { }) .sum::() / num_validators; - let average_tps = average_transactions as f64 / duration_secs as f64; + let cutoff_transactions = transaction_outcome + .map(|outcome| outcome.cutoff_committed_transactions) + .unwrap_or(average_transactions); + let average_tps = cutoff_transactions as f64 / duration_secs as f64; let average_blocks_submitted = metrics .iter() .enumerate() .map(|(index, metrics)| { - metrics.dag_state_entries.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.dag_state_entries) - .unwrap_or_default(), - ) + counter_cutoffs + .as_ref() + .and_then(|cutoffs| cutoffs.get(index)) + .map(|cutoff| cutoff.dag_state_entries) + .unwrap_or_else(|| metrics.dag_state_entries.get()) + .saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.dag_state_entries) + .unwrap_or_default(), + ) }) .sum::() / num_validators; @@ -1572,13 +2003,18 @@ impl Metrics { .iter() .enumerate() .map(|(index, metrics)| { - metrics.bytes_sent_total.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.bytes_sent) - .unwrap_or_default(), - ) + counter_cutoffs + .as_ref() + .and_then(|cutoffs| cutoffs.get(index)) + .map(|cutoff| cutoff.bytes_sent) + .unwrap_or_else(|| metrics.bytes_sent_total.get()) + .saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.bytes_sent) + .unwrap_or_default(), + ) }) .sum::() / num_validators; @@ -1586,13 +2022,18 @@ impl Metrics { .iter() .enumerate() .map(|(index, metrics)| { - metrics.bytes_received_total.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.bytes_received) - .unwrap_or_default(), - ) + counter_cutoffs + .as_ref() + .and_then(|cutoffs| cutoffs.get(index)) + .map(|cutoff| cutoff.bytes_received) + .unwrap_or_else(|| metrics.bytes_received_total.get()) + .saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.bytes_received) + .unwrap_or_default(), + ) }) .sum::() / num_validators; @@ -1679,13 +2120,43 @@ impl Metrics { table.add_row(row![bH2->""]); table.add_row(row![bH2->"Performance Metrics"]); table.add_row( - row![b->"Average block latency:", format!("{:.2} millis", p50_block_committed_latency)], + row![b->"p50 block latency:", format!("{:.2} millis", p50_block_committed_latency)], ); table.add_row(row![ - b->"Average e2e latency:", + b->"p50 e2e latency:", format!("{:.2} millis", p50_transaction_committed_latency) ]); - table.add_row(row![b->"Average TPS:", format!("{:.2} tx/s", average_tps)]); + if let Some(outcome) = transaction_outcome { + table.add_row(row![ + b->"Offered TPS:", + format!( + "{:.2} tx/s ({} exact successful submissions)", + outcome.offered_transactions as f64 / duration_secs as f64, + outcome.offered_transactions, + ) + ]); + table.add_row(row![ + b->"Committed TPS at cutoff:", + format!("{:.2} tx/s", outcome.cutoff_committed_transactions as f64 / duration_secs as f64) + ]); + table.add_row(row![ + b->"Eventual active-window TPS:", + format!( + "{:.2} tx/s ({}, drain {:.2}s)", + outcome.eventual_committed_transactions as f64 / duration_secs as f64, + if outcome.drain_complete { "complete" } else { "INCOMPLETE" }, + outcome.drain_elapsed.as_secs_f64(), + ) + ]); + table.add_row(row![ + b->"Cutoff backlog:", + outcome + .offered_transactions + .saturating_sub(outcome.cutoff_committed_transactions) + ]); + } else { + table.add_row(row![b->"Average TPS:", format!("{:.2} tx/s", average_tps)]); + } table.add_row(row![b->"Average BPS:", format!("{:.2} blocks/s", average_bps)]); // Network metrics @@ -1709,10 +2180,17 @@ impl Metrics { .iter() .enumerate() .map(|(validator_index, metrics)| { - let current = metrics - .network_message_bytes_sent_total - .with_label_values(&[request_type]) - .get(); + let current = counter_cutoffs + .as_ref() + .and_then(|cutoffs| cutoffs.get(validator_index)) + .and_then(|cutoff| cutoff.outbound_messages.get(message_index)) + .map(|(bytes, _)| *bytes) + .unwrap_or_else(|| { + metrics + .network_message_bytes_sent_total + .with_label_values(&[request_type]) + .get() + }); let baseline = counter_baselines .as_ref() .and_then(|baselines| baselines.get(validator_index)) @@ -1730,10 +2208,17 @@ impl Metrics { .iter() .enumerate() .map(|(validator_index, metrics)| { - let current = metrics - .network_requests_sent_total - .with_label_values(&[request_type]) - .get(); + let current = counter_cutoffs + .as_ref() + .and_then(|cutoffs| cutoffs.get(validator_index)) + .and_then(|cutoff| cutoff.outbound_messages.get(message_index)) + .map(|(_, requests)| *requests) + .unwrap_or_else(|| { + metrics + .network_requests_sent_total + .with_label_values(&[request_type]) + .get() + }); let baseline = counter_baselines .as_ref() .and_then(|baselines| baselines.get(validator_index)) @@ -1765,8 +2250,14 @@ impl Metrics { } } let total_average_transactions = (average_tps * duration_secs as f64) as u64; - let bandwidth_efficiency = if total_average_transactions > 0 { - average_bytes_sent as f64 / total_average_transactions as f64 / 512.0 + // Every honest validator sequences the same global offered set, so + // its bandwidth denominator is the aggregate submissions across all + // generators—not one generator's local share. + let offered_global = transaction_outcome + .map(|outcome| outcome.offered_transactions as f64) + .unwrap_or(total_average_transactions as f64); + let bandwidth_efficiency = if offered_global > 0.0 { + average_bytes_sent as f64 / offered_global / 512.0 } else { 0.0 }; @@ -1777,6 +2268,7 @@ impl Metrics { &metrics, committee_size, autonomous_clock_baselines.as_deref(), + autonomous_clock_cutoffs.as_deref(), starfish_rbc_dag_embedded_rbc_authority_expected, ); let round_lag = summary.maximum_round.saturating_sub(summary.minimum_round); @@ -1790,7 +2282,7 @@ impl Metrics { } ]); table.add_row(row![ - b->"Clock verdict:", + b->"Cutoff clock verdict:", if summary.verdict_valid { "VALID".to_owned() } else { @@ -1798,7 +2290,7 @@ impl Metrics { } ]); table.add_row(row![ - b->"Valid/progress/bounded validators:", + b->"Cutoff valid/progress/bounded validators:", format!( "{}/{}, {}/{}, {}/{}", summary.valid_nodes, @@ -1810,7 +2302,7 @@ impl Metrics { ) ]); table.add_row(row![ - b->"Clock/WAL progress:", + b->"Cutoff clock/WAL progress:", format!( "heartbeats={}, carrier deliveries={}, application deliveries={}, committed frontiers={}, frontier applications={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, @@ -1827,7 +2319,7 @@ impl Metrics { ) ]); table.add_row(row![ - b->"Bounded live state:", + b->"Cutoff bounded state:", format!( "round skew={round_lag}/{}, max phase backlog={}/{}, admitted authors={}/{}, stake={}, max buffered={}/{}, pending recovery={}", STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, @@ -1841,6 +2333,328 @@ impl Metrics { summary.pending_recovery, ) ]); + let cutoff_per_validator_progress = metrics + .iter() + .enumerate() + .map(|(index, metrics)| { + let baseline = autonomous_clock_baselines + .as_deref() + .and_then(|baselines| baselines.get(index)) + .copied() + .unwrap_or_default(); + let snapshot = autonomous_clock_cutoffs + .as_deref() + .and_then(|snapshots| snapshots.get(index)) + .copied() + .unwrap_or_else(|| metrics.autonomous_clock_benchmark_snapshot()); + let applications = snapshot + .delivered_applications + .saturating_sub(baseline.delivered_applications); + let frontiers = snapshot + .committed_frontiers + .saturating_sub(baseline.committed_frontiers); + format!( + "{index}:r{}/a{applications}/f{frontiers}/v{}", + snapshot.carrier_round, snapshot.clock_valid, + ) + }) + .collect::>() + .join(", "); + table.add_row(row![ + b->"Cutoff per-validator round/app/frontier/valid:", + cutoff_per_validator_progress, + ]); + let final_per_validator_state = metrics + .iter() + .enumerate() + .map(|(index, metrics)| { + let snapshot = metrics.autonomous_clock_benchmark_snapshot(); + format!( + "{index}:r{}/q{}/b{}/rec{}/v{}", + snapshot.carrier_round, + snapshot.phase_backlog, + snapshot.buffered_authenticated, + snapshot.pending_recovery, + snapshot.clock_valid, + ) + }) + .collect::>() + .join(", "); + table.add_row(row![ + b->"Final/drain state (diagnostic only):", + final_per_validator_state, + ]); + let shadow_input_count = |kind: &str, outcome: &str| { + metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&[kind, outcome]) + .get() + }) + .sum::() + }; + table.add_row(row![ + b->"Carrier transport/ingress:", + format!( + "network sent/disconnected/dropped={}/{}/{}, carriers authenticated/retained/rejected={}/{}/{}, sync requests sent/served={}/{}, sync responses authenticated/rejected={}/{}, subscriptions to/by peers={}..{}/{}..{}", + shadow_input_count("network", "sent"), + shadow_input_count("network", "disconnected"), + shadow_input_count("network", "dropped_backpressure"), + shadow_input_count("carrier", "authenticated"), + shadow_input_count("carrier", "retained_unauthenticated"), + shadow_input_count("carrier", "rejected"), + shadow_input_count("carrier_sync_request", "sent"), + shadow_input_count("carrier_sync_request", "served"), + shadow_input_count("carrier_sync_response", "authenticated"), + shadow_input_count("carrier_sync_response", "rejected"), + metrics + .iter() + .map(|metrics| metrics.subscribed_to_peers.get()) + .min() + .unwrap_or_default(), + metrics + .iter() + .map(|metrics| metrics.subscribed_to_peers.get()) + .max() + .unwrap_or_default(), + metrics + .iter() + .map(|metrics| metrics.subscribed_by_peers.get()) + .min() + .unwrap_or_default(), + metrics + .iter() + .map(|metrics| metrics.subscribed_by_peers.get()) + .max() + .unwrap_or_default(), + ) + ]); + let stage_latency = RBC_DAG_PIPELINE_LATENCY_STAGES + .iter() + .map(|stage| { + let total = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_pipeline_latency_ns_total + .with_label_values(&[stage]) + .get() + }) + .sum::(); + let samples = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[stage]) + .get() + }) + .sum::(); + let maximum = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_pipeline_latency_ns_max + .with_label_values(&[stage]) + .get() + }) + .max() + .unwrap_or_default(); + let average_ms = if samples == 0 { + 0.0 + } else { + total as f64 / samples as f64 / 1_000_000.0 + }; + format!( + "{}={average_ms:.1}/{:.1}ms(n={samples})", + stage.strip_prefix("creation_to_").unwrap_or(stage), + maximum as f64 / 1_000_000.0, + ) + }) + .collect::>() + .join(", "); + table.add_row(row![ + b->"Pipeline latency avg/max:", + stage_latency + ]); + + let commit_distance = |kind: &str| { + let total = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_commit_distance_rounds_total + .with_label_values(&[kind]) + .get() + }) + .sum::(); + let samples = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_commit_distance_samples_total + .with_label_values(&[kind]) + .get() + }) + .sum::(); + let maximum = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_commit_distance_rounds_max + .with_label_values(&[kind]) + .get() + }) + .max() + .unwrap_or_default(); + format_rbc_dag_round_distance(total, samples, maximum) + }; + table.add_row(row![ + b->"Commit distance avg/max:", + format!( + "physical forward={}, backward={}", + commit_distance(RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD), + commit_distance(RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD), + ) + ]); + + let queue_depth = |queue: &str, maximum: bool| { + metrics + .iter() + .map(|metrics| { + if maximum { + metrics + .starfish_rbc_dag_pipeline_queue_depth_max + .with_label_values(&[queue]) + .get() + } else { + metrics + .starfish_rbc_dag_pipeline_queue_depth + .with_label_values(&[queue]) + .get() + } + }) + .max() + .unwrap_or_default() + }; + let highest_projected = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_highest_projected_consensus_round + .get() + }) + .max() + .unwrap_or_default(); + let next_undecided = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_next_undecided_consensus_round + .get() + }) + .min() + .unwrap_or_default(); + let next_undecided_stake = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_next_undecided_projected_stake + .get() + }) + .min() + .unwrap_or_default(); + let last_committed = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_last_committed_consensus_round + .get() + }) + .min() + .unwrap_or_default(); + let hol = RBC_DAG_PROJECTION_HOL_STATES + .iter() + .filter_map(|reason| { + let nodes = metrics + .iter() + .filter(|metrics| { + metrics + .starfish_rbc_dag_projection_hol_state + .with_label_values(&[reason]) + .get() + == 1 + }) + .count(); + (nodes != 0).then(|| format!("{reason}:{nodes}")) + }) + .collect::>() + .join(","); + table.add_row(row![ + b->"Projection/HOL state:", + format!( + "pending local={}/{}, projection={}/{}, highest projected={highest_projected}, oldest undecided={next_undecided} (lag={}, projected stake={next_undecided_stake}), last committed={last_committed} (lag={}), HOL=[{hol}]", + queue_depth("local", false), + queue_depth("local", true), + queue_depth("projection", false), + queue_depth("projection", true), + highest_projected.saturating_sub(next_undecided), + highest_projected.saturating_sub(last_committed), + ) + ]); + let decision_count = |outcome: &str| { + metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&[outcome]) + .get() + }) + .sum::() + }; + table.add_row(row![ + b->"Projection decisions:", + format!( + "direct commit/skip={}/{}, indirect commit/skip={}/{}, undecided={}", + decision_count("direct_commit"), + decision_count("direct_skip"), + decision_count("indirect_commit"), + decision_count("indirect_skip"), + decision_count("undecided"), + ) + ]); + let frontier_count = |stage: &str| { + metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_dag_frontier_events_total + .with_label_values(&[stage]) + .get() + }) + .sum::() + }; + table.add_row(row![ + b->"Frontier event application:", + format!( + "generated={}, applied={}, ignored={}, inflight current/max={}/{}", + frontier_count("generated"), + frontier_count("applied"), + frontier_count("ignored"), + metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_frontiers_inflight.get()) + .sum::(), + metrics + .iter() + .map(|metrics| metrics.starfish_rbc_dag_frontiers_inflight_max.get()) + .max() + .unwrap_or_default(), + ) + ]); } else if starfish_rbc_dag_shadow_expected { let valid_nodes = metrics .iter() @@ -2215,6 +3029,29 @@ impl MetricReporter { self.block_bundle_size_bytes.lock().clear_receive_all(); self.connection_latency.lock().clear_receive_all(); } + + /// Discard every sample produced before a coordinated benchmark window. + /// `clear_receive_all` intentionally preserves newly received points for + /// periodic reporting; a benchmark reset needs the opposite order. + pub fn reset_for_benchmark_window(&self) { + fn drain_then_clear(histogram: &mut PreciseHistogram) + where + T: Ord + AddAssign + DivUsize + Copy + Default, + { + histogram.receive_all(); + histogram.reset(); + } + + drain_then_clear(&mut self.transaction_committed_latency.lock().histogram); + drain_then_clear(&mut self.block_committed_latency.lock().histogram); + drain_then_clear(&mut self.proposed_block_size_bytes.lock().histogram); + drain_then_clear(&mut self.proposed_header_size_bytes.lock().histogram); + drain_then_clear(&mut self.proposed_transaction_size_bytes.lock().histogram); + drain_then_clear(&mut self.block_bundle_size_bytes.lock().histogram); + for (histogram, _) in &mut self.connection_latency.lock().histograms { + drain_then_clear(histogram); + } + } } pub fn print_network_address_table(addresses: &[SocketAddr]) { @@ -2341,6 +3178,22 @@ mod tests { .starfish_rbc_dag_projection_decisions_total .with_label_values(&["direct_commit"]) .inc(); + metrics.metrics_active.store(true, Ordering::Relaxed); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, + 12, + 2, + 8, + ); + metrics.observe_starfish_rbc_dag_commit_round_distance( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, + 7, + 2, + 5, + ); + metrics.set_starfish_rbc_dag_pipeline_state(2, 3, 9, 7, 3, 6, "awaiting_indirect_anchor"); + metrics.starfish_rbc_dag_frontier_generated(); + metrics.starfish_rbc_dag_frontier_applied(); let gathered = registry.gather(); for name in [ @@ -2365,6 +3218,22 @@ mod tests { "starfish_rbc_dag_shadow_buffered_authenticated", "starfish_rbc_dag_projected_vertices_total", "starfish_rbc_dag_projection_decisions_total", + "starfish_rbc_dag_pipeline_latency_ns_total", + "starfish_rbc_dag_pipeline_latency_samples_total", + "starfish_rbc_dag_pipeline_latency_ns_max", + "starfish_rbc_dag_commit_distance_rounds_total", + "starfish_rbc_dag_commit_distance_samples_total", + "starfish_rbc_dag_commit_distance_rounds_max", + "starfish_rbc_dag_pipeline_queue_depth", + "starfish_rbc_dag_pipeline_queue_depth_max", + "starfish_rbc_dag_highest_projected_consensus_round", + "starfish_rbc_dag_next_undecided_consensus_round", + "starfish_rbc_dag_next_undecided_projected_stake", + "starfish_rbc_dag_last_committed_consensus_round", + "starfish_rbc_dag_projection_hol_state", + "starfish_rbc_dag_frontier_events_total", + "starfish_rbc_dag_frontiers_inflight", + "starfish_rbc_dag_frontiers_inflight_max", ] { assert!( gathered.iter().any(|family| family.get_name() == name), @@ -2373,6 +3242,194 @@ mod tests { } } + #[test] + fn rbc_dag_pipeline_latency_drains_while_protocol_distance_closes_at_cutoff() { + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new(®istry, None, None, None); + + metrics.metrics_active.store(false, Ordering::Relaxed); + metrics + .transaction_metrics_active + .store(false, Ordering::Relaxed); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + 99, + 1, + 99, + ); + metrics.observe_starfish_rbc_dag_commit_round_distance( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, + 99, + 1, + 99, + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) + .get(), + 0 + ); + assert_eq!( + metrics + .starfish_rbc_dag_commit_distance_samples_total + .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD]) + .get(), + 0 + ); + + // The submission window is closed but application observation remains + // open for the bounded drain. Only application pipeline latency may + // advance; protocol round-distance rates stay frozen at cutoff. + metrics + .transaction_metrics_active + .store(true, Ordering::Relaxed); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + 11, + 1, + 11, + ); + metrics.observe_starfish_rbc_dag_commit_round_distance( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, + 99, + 1, + 99, + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) + .get(), + 1 + ); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, + 13, + 1, + 13, + ); + assert_eq!( + metrics.starfish_rbc_dag_frontier_applied_latency_samples(), + 1 + ); + assert_eq!( + metrics.starfish_rbc_dag_frontier_applied_sequenced_transactions(), + 0, + "latency alone must not publish the ordered drain acknowledgement" + ); + metrics.sequenced_transactions_total.inc_by(17); + metrics.starfish_rbc_dag_frontier_applied(); + assert_eq!( + metrics.starfish_rbc_dag_frontier_applied_sequenced_transactions(), + 17, + "frontier application must acknowledge all transactions sequenced before its final latency observation" + ); + assert_eq!( + metrics + .starfish_rbc_dag_commit_distance_samples_total + .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD]) + .get(), + 0 + ); + + metrics.metrics_active.store(true, Ordering::Relaxed); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + 30, + 2, + 20, + ); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + 4, + 1, + 4, + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_latency_ns_total + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) + .get(), + 45 + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) + .get(), + 4 + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_latency_ns_max + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) + .get(), + 20 + ); + + metrics.observe_starfish_rbc_dag_commit_round_distance( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, + 7, + 2, + 5, + ); + metrics.observe_starfish_rbc_dag_commit_round_distance( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, + 3, + 1, + 3, + ); + assert_eq!( + metrics + .starfish_rbc_dag_commit_distance_rounds_total + .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD]) + .get(), + 7 + ); + assert_eq!( + metrics + .starfish_rbc_dag_commit_distance_samples_total + .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD]) + .get(), + 1 + ); + assert_eq!( + format_rbc_dag_round_distance(11, 2, 6), + "5.50/6 rounds (n=2)" + ); + + metrics.set_starfish_rbc_dag_pipeline_state(4, 5, 12, 10, 2, 8, "insufficient_lookahead"); + metrics.set_starfish_rbc_dag_pipeline_state(1, 2, 13, 11, 4, 9, "ready"); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_queue_depth_max + .with_label_values(&["local"]) + .get(), + 4 + ); + assert_eq!( + metrics + .starfish_rbc_dag_projection_hol_state + .with_label_values(&["ready"]) + .get(), + 1 + ); + assert_eq!( + metrics + .starfish_rbc_dag_projection_hol_state + .with_label_values(&["insufficient_lookahead"]) + .get(), + 0 + ); + + metrics.starfish_rbc_dag_frontier_generated(); + metrics.starfish_rbc_dag_frontier_generated(); + metrics.starfish_rbc_dag_frontier_applied(); + assert_eq!(metrics.starfish_rbc_dag_frontiers_inflight.get(), 1); + assert_eq!(metrics.starfish_rbc_dag_frontiers_inflight_max.get(), 2); + } + fn autonomous_clock_metrics( round: i64, phase_backlog: i64, @@ -2427,7 +3484,7 @@ mod tests { autonomous_clock_metrics(11, 6, 1), ]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, None, false); assert!(summary.verdict_valid); assert_eq!(summary.valid_nodes, 4); @@ -2441,6 +3498,39 @@ mod tests { assert_eq!(summary.maximum_round, 11); } + #[test] + fn autonomous_clock_verdict_uses_cutoff_snapshot_not_drain_state() { + let metrics = vec![ + autonomous_clock_metrics(8, 0, 0), + autonomous_clock_metrics(20, 0, 0), + ]; + let invalid_cutoff = metrics + .iter() + .map(|metrics| metrics.autonomous_clock_benchmark_snapshot()) + .collect::>(); + + // A later drain-time convergence is useful diagnostic state, but it + // cannot repair the measured interval's twelve-round cutoff skew. + metrics[1].starfish_rbc_dag_shadow_carrier_round.set(9); + assert!(summarize_autonomous_clock_benchmark(&metrics, 2, None, None, false).verdict_valid); + assert!( + !summarize_autonomous_clock_benchmark(&metrics, 2, None, Some(&invalid_cutoff), false,) + .verdict_valid + ); + + let valid_cutoff = metrics + .iter() + .map(|metrics| metrics.autonomous_clock_benchmark_snapshot()) + .collect::>(); + metrics[1].starfish_rbc_dag_shadow_clock_valid.set(0); + metrics[1].starfish_rbc_dag_shadow_pending_recovery.set(1); + assert!( + summarize_autonomous_clock_benchmark(&metrics, 2, None, Some(&valid_cutoff), false,) + .verdict_valid, + "drain-time invalidity must remain diagnostic rather than rewriting cutoff validity" + ); + } + #[test] fn autonomous_clock_summary_requires_progress_after_the_benchmark_baseline() { let metrics = vec![ @@ -2453,7 +3543,7 @@ mod tests { .collect::>(); assert!( - !summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) + !summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), None, false) .verdict_valid ); @@ -2487,7 +3577,7 @@ mod tests { } assert!( - summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) + summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), None, false) .verdict_valid ); } @@ -2526,6 +3616,7 @@ mod tests { &[Arc::clone(metrics)], 2, Some(&baselines), + None, true, ) .verdict_valid @@ -2548,6 +3639,7 @@ mod tests { &[Arc::clone(metrics)], 2, Some(&baselines), + None, true, ) .verdict_valid @@ -2566,11 +3658,11 @@ mod tests { let unbounded = autonomous_clock_metrics( 20, STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR * 4 + 1, - STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR * 4 + 1, + starfish_rbc_dag_autonomous_buffered_settled_bound(4) + 1, ); let metrics = vec![no_progress, invalid_clock, unbounded]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, None, false); assert!(!summary.verdict_valid); assert_eq!(summary.valid_nodes, 2); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 37e39241..e2789e11 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -4,22 +4,24 @@ use std::{ collections::{HashMap, VecDeque}, + fmt, + panic::AssertUnwindSafe, path::PathBuf, sync::{ Arc, - atomic::{AtomicU32, Ordering}, + atomic::{AtomicBool, AtomicU32, Ordering}, }, - time::Duration, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use ahash::{AHashMap, AHashSet}; -use futures::future::join_all; +use futures::{FutureExt, future::join_all}; use rand::seq::SliceRandom; use reed_solomon_simd::ReedSolomonEncoder; use tokio::time::Instant; use tokio::{ select, - sync::{Notify, mpsc}, + sync::{Notify, Semaphore, mpsc, watch}, }; use crate::{ @@ -35,10 +37,14 @@ use crate::{ }, core::Core, core_thread::CoreThreadDispatcher, - crypto::{Blake3Hasher, BlsSigner, MacKey}, + crypto::{Blake3Hasher, BlsSigner, MacKey, TransactionsCommitment}, dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, - metrics::{Metrics, UtilizationTimerVecExt}, + metrics::{ + Metrics, RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, + UtilizationTimerVecExt, + }, network::{BlockBatch, Connection, Network, NetworkMessage, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, sailfish_service::{ @@ -47,16 +53,20 @@ use crate::{ shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader, RbcCommitteeId, RbcProtocolInstanceId}, starfish_rbc_dag::{ - RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, - projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, + CandidateCarrierV1, MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, + RbcDagProtocolInstanceId, projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ - ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, + CommittedApplicationDiagnosticV1, CommittedFrontierDeltaV1, ShadowAuthorizerV1, + ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, }, starfish_rbc_dag_shadow_service::{ - ShadowServiceErrorV1, ShadowServiceEventV1, StarfishRbcDagShadowServiceHandleV1, - start_starfish_rbc_dag_autonomous_clock_service_v1, - start_starfish_rbc_dag_shadow_service_v1, + ShadowApplicationAuthorizationBasisV1, ShadowServiceErrorV1, ShadowServiceEventV1, + StarfishRbcDagShadowServiceHandleV1, + start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1, + start_starfish_rbc_dag_autonomous_clock_service_paused_with_metrics_v1, + start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1, + start_starfish_rbc_dag_shadow_service_with_metrics_v1, }, starfish_rbc_service::{ RbcInitialAuthenticator, RbcPhaseAuthorityV1, RbcServiceEvent, RbcServiceHandle, @@ -66,7 +76,7 @@ use crate::{ types::{ AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, - RoundNumber, TransactionData, VerifiedBlock, format_authority_index, + RoundNumber, TimestampNs, TransactionData, VerifiedBlock, format_authority_index, }, }; @@ -75,9 +85,525 @@ const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); const STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); +const STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); +const STARFISH_RBC_DAG_CORE_CONTROL_CAPACITY: usize = 64; +const STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY: usize = 64; +// Priority entries contain at most one carrier or application payload plus a +// bounded authentication/frame reserve. Proactive carriers may contain both. +// Count and byte accounting are independent so a future wire-size regression +// cannot turn the fixed key caps into an unbounded memory reservoir. +const STARFISH_RBC_DAG_OUTBOUND_FRAME_RESERVE: usize = 128 * 1024; +const STARFISH_RBC_DAG_OUTBOUND_PRIORITY_ENTRY_BYTES: usize = + MAX_CARRIER_CONTENT_SIZE_V1 + STARFISH_RBC_DAG_OUTBOUND_FRAME_RESERVE; +const STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_ENTRY_BYTES: usize = + MAX_CARRIER_CONTENT_SIZE_V1 * 2 + STARFISH_RBC_DAG_OUTBOUND_FRAME_RESERVE; +const STARFISH_RBC_DAG_OUTBOUND_PRIORITY_BYTES: usize = + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY * MAX_CARRIER_CONTENT_SIZE_V1; +const STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_BYTES: usize = + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY * MAX_CARRIER_CONTENT_SIZE_V1; const STARFISH_RBC_DAG_AUTONOMOUS_INSTANCE_CONTEXT: &str = "STARFISH_RBC_DAG_AUTONOMOUS_CLOCK_V1_PROTOCOL_INSTANCE"; +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +enum RbcDagOutboundKeyV1 { + Proactive(BlockReference), + CarrierRequest(BlockReference), + CarrierResponse(BlockReference), + SyncRequest(AuthorityIndex, RoundNumber), + SyncResponse(AuthorityIndex, RoundNumber), + ApplicationPayloadRequest(BlockReference), + ApplicationPayloadResponse(BlockReference), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RbcDagOutboundClassV1 { + Priority, + Proactive, +} + +impl RbcDagOutboundClassV1 { + fn label(self) -> &'static str { + match self { + Self::Priority => "priority", + Self::Proactive => "proactive", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RbcDagOutboundEnqueueV1 { + Added, + Coalesced, +} + +#[derive(Debug)] +enum RbcDagOutboundMailboxErrorV1 { + Unsupported, + InvalidProactive(String), + Serialization(String), + EntryTooLarge { + class: RbcDagOutboundClassV1, + actual: usize, + maximum: usize, + }, + KeyCapacity { + class: RbcDagOutboundClassV1, + capacity: usize, + }, + ByteCapacity { + class: RbcDagOutboundClassV1, + attempted: usize, + capacity: usize, + }, + ConflictingDuplicate { + class: RbcDagOutboundClassV1, + key: RbcDagOutboundKeyV1, + }, + DownstreamSaturated(RbcDagOutboundClassV1), + DownstreamClosed, + Failed(String), +} + +impl fmt::Display for RbcDagOutboundMailboxErrorV1 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unsupported => write!(formatter, "unsupported non-RBC-DAG outbound message"), + Self::InvalidProactive(error) => { + write!(formatter, "invalid proactive carrier: {error}") + } + Self::Serialization(error) => { + write!(formatter, "outbound message serialization failed: {error}") + } + Self::EntryTooLarge { + class, + actual, + maximum, + } => write!( + formatter, + "{} outbound entry is {actual} bytes, maximum {maximum}", + class.label(), + ), + Self::KeyCapacity { class, capacity } => write!( + formatter, + "{} outbound key capacity {capacity} exhausted", + class.label(), + ), + Self::ByteCapacity { + class, + attempted, + capacity, + } => write!( + formatter, + "{} outbound byte capacity {capacity} exhausted by {attempted} bytes", + class.label(), + ), + Self::ConflictingDuplicate { class, key } => write!( + formatter, + "conflicting {} outbound duplicate for {key:?}", + class.label(), + ), + Self::DownstreamSaturated(class) => { + write!( + formatter, + "downstream {} network channel saturated", + class.label() + ) + } + Self::DownstreamClosed => write!(formatter, "downstream network sender closed"), + Self::Failed(reason) => write!(formatter, "outbound mailbox already failed: {reason}"), + } + } +} + +struct RbcDagOutboundEntryV1 { + message: NetworkMessage, + framed_bytes: usize, +} + +struct RbcDagOutboundLaneV1 { + order: VecDeque, + entries: AHashMap, + bytes: usize, + key_capacity: usize, + byte_capacity: usize, + entry_byte_capacity: usize, +} + +impl RbcDagOutboundLaneV1 { + fn new(key_capacity: usize, byte_capacity: usize, entry_byte_capacity: usize) -> Self { + Self { + order: VecDeque::new(), + entries: AHashMap::new(), + bytes: 0, + key_capacity, + byte_capacity, + entry_byte_capacity, + } + } + + fn enqueue( + &mut self, + class: RbcDagOutboundClassV1, + key: RbcDagOutboundKeyV1, + entry: RbcDagOutboundEntryV1, + ) -> Result { + if let Some(existing) = self.entries.get(&key) { + return if rbc_dag_outbound_messages_equal(&existing.message, &entry.message) { + Ok(RbcDagOutboundEnqueueV1::Coalesced) + } else { + Err(RbcDagOutboundMailboxErrorV1::ConflictingDuplicate { class, key }) + }; + } + if entry.framed_bytes > self.entry_byte_capacity { + return Err(RbcDagOutboundMailboxErrorV1::EntryTooLarge { + class, + actual: entry.framed_bytes, + maximum: self.entry_byte_capacity, + }); + } + if self.entries.len() >= self.key_capacity { + return Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { + class, + capacity: self.key_capacity, + }); + } + let attempted = self.bytes.checked_add(entry.framed_bytes).ok_or( + RbcDagOutboundMailboxErrorV1::ByteCapacity { + class, + attempted: usize::MAX, + capacity: self.byte_capacity, + }, + )?; + if attempted > self.byte_capacity { + return Err(RbcDagOutboundMailboxErrorV1::ByteCapacity { + class, + attempted, + capacity: self.byte_capacity, + }); + } + self.bytes = attempted; + self.order.push_back(key); + assert!(self.entries.insert(key, entry).is_none()); + Ok(RbcDagOutboundEnqueueV1::Added) + } + + fn pop_front(&mut self) -> Option { + let key = self.order.pop_front()?; + let entry = self + .entries + .remove(&key) + .expect("queued RBC-DAG outbound key retains its exact entry"); + self.bytes = self + .bytes + .checked_sub(entry.framed_bytes) + .expect("RBC-DAG outbound byte accounting cannot underflow"); + Some(entry.message) + } +} + +struct RbcDagOutboundMailboxStateV1 { + priority: RbcDagOutboundLaneV1, + proactive: RbcDagOutboundLaneV1, + failure: Option, +} + +impl RbcDagOutboundMailboxStateV1 { + fn production() -> Self { + Self { + priority: RbcDagOutboundLaneV1::new( + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + STARFISH_RBC_DAG_OUTBOUND_PRIORITY_BYTES, + STARFISH_RBC_DAG_OUTBOUND_PRIORITY_ENTRY_BYTES, + ), + proactive: RbcDagOutboundLaneV1::new( + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_BYTES, + STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_ENTRY_BYTES, + ), + failure: None, + } + } +} + +struct RbcDagOutboundMailboxInnerV1 { + state: parking_lot::Mutex, + notified: Notify, +} + +#[derive(Clone)] +struct RbcDagOutboundMailboxV1 { + inner: Arc, +} + +impl RbcDagOutboundMailboxV1 { + fn new() -> Self { + Self::from_state(RbcDagOutboundMailboxStateV1::production()) + } + + fn from_state(state: RbcDagOutboundMailboxStateV1) -> Self { + Self { + inner: Arc::new(RbcDagOutboundMailboxInnerV1 { + state: parking_lot::Mutex::new(state), + notified: Notify::new(), + }), + } + } + + fn enqueue( + &self, + message: NetworkMessage, + committee: &Committee, + ) -> Result { + if let Some(reason) = self.inner.state.lock().failure.clone() { + return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); + } + let (class, key) = match rbc_dag_outbound_classification(&message, committee) { + Ok(classification) => classification, + Err(error) => { + self.fail(&error); + return Err(error); + } + }; + let framed_bytes = match bincode::serialized_size(&message) + .map_err(|error| RbcDagOutboundMailboxErrorV1::Serialization(error.to_string())) + .and_then(|size| { + usize::try_from(size) + .ok() + .and_then(|size| size.checked_add(4)) + .ok_or_else(|| { + RbcDagOutboundMailboxErrorV1::Serialization( + "framed size does not fit usize".to_owned(), + ) + }) + }) { + Ok(framed_bytes) => framed_bytes, + Err(error) => { + self.fail(&error); + return Err(error); + } + }; + let entry = RbcDagOutboundEntryV1 { + message, + framed_bytes, + }; + let mut state = self.inner.state.lock(); + if let Some(reason) = state.failure.clone() { + return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); + } + let result = match class { + RbcDagOutboundClassV1::Priority => state.priority.enqueue(class, key, entry), + RbcDagOutboundClassV1::Proactive => state.proactive.enqueue(class, key, entry), + }; + match result { + Ok(outcome) => { + drop(state); + if outcome == RbcDagOutboundEnqueueV1::Added { + self.inner.notified.notify_one(); + } + Ok(outcome) + } + Err(error) => { + state.failure = Some(error.to_string()); + drop(state); + self.inner.notified.notify_waiters(); + Err(error) + } + } + } + + fn fail(&self, error: &RbcDagOutboundMailboxErrorV1) { + let mut state = self.inner.state.lock(); + state.failure.get_or_insert_with(|| error.to_string()); + drop(state); + self.inner.notified.notify_waiters(); + } + + fn try_pop(&self) -> Option<(RbcDagOutboundClassV1, NetworkMessage)> { + let mut state = self.inner.state.lock(); + if state.failure.is_some() { + return None; + } + state + .priority + .pop_front() + .map(|message| (RbcDagOutboundClassV1::Priority, message)) + .or_else(|| { + state + .proactive + .pop_front() + .map(|message| (RbcDagOutboundClassV1::Proactive, message)) + }) + } + + async fn recv(&self) -> Option<(RbcDagOutboundClassV1, NetworkMessage)> { + loop { + let notified = self.inner.notified.notified(); + if let Some(message) = self.try_pop() { + return Some(message); + } + if self.inner.state.lock().failure.is_some() { + return None; + } + notified.await; + } + } +} + +fn rbc_dag_outbound_classification( + message: &NetworkMessage, + committee: &Committee, +) -> Result<(RbcDagOutboundClassV1, RbcDagOutboundKeyV1), RbcDagOutboundMailboxErrorV1> { + let priority = RbcDagOutboundClassV1::Priority; + match message { + NetworkMessage::RbcDagShadowCarrier(carrier) => { + let candidate = + CandidateCarrierV1::decode_wire(&carrier.canonical_carrier, committee, None) + .map_err(|error| { + RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) + })?; + let canonical = candidate.canonical_wire_bytes().map_err(|error| { + RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) + })?; + if canonical != carrier.canonical_carrier { + return Err(RbcDagOutboundMailboxErrorV1::InvalidProactive( + "non-canonical carrier wire".to_owned(), + )); + } + Ok(( + RbcDagOutboundClassV1::Proactive, + RbcDagOutboundKeyV1::Proactive(candidate.reference()), + )) + } + NetworkMessage::RbcDagShadowCarrierRequest(reference) => { + Ok((priority, RbcDagOutboundKeyV1::CarrierRequest(*reference))) + } + NetworkMessage::RbcDagShadowCarrierResponse(response) => Ok(( + priority, + RbcDagOutboundKeyV1::CarrierResponse(response.reference), + )), + NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => Ok(( + priority, + RbcDagOutboundKeyV1::SyncRequest(request.author, request.round), + )), + NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => Ok(( + priority, + RbcDagOutboundKeyV1::SyncResponse(response.author, response.round), + )), + NetworkMessage::RbcDagApplicationPayloadRequest(application) => Ok(( + priority, + RbcDagOutboundKeyV1::ApplicationPayloadRequest(*application), + )), + NetworkMessage::RbcDagApplicationPayloadResponse(response) => Ok(( + priority, + RbcDagOutboundKeyV1::ApplicationPayloadResponse(response.application), + )), + _ => Err(RbcDagOutboundMailboxErrorV1::Unsupported), + } +} + +fn rbc_dag_outbound_messages_equal(left: &NetworkMessage, right: &NetworkMessage) -> bool { + match (left, right) { + (NetworkMessage::RbcDagShadowCarrier(left), NetworkMessage::RbcDagShadowCarrier(right)) => { + left == right + } + ( + NetworkMessage::RbcDagShadowCarrierRequest(left), + NetworkMessage::RbcDagShadowCarrierRequest(right), + ) => left == right, + ( + NetworkMessage::RbcDagShadowCarrierResponse(left), + NetworkMessage::RbcDagShadowCarrierResponse(right), + ) => left == right, + ( + NetworkMessage::RbcDagShadowCarrierSyncRequest(left), + NetworkMessage::RbcDagShadowCarrierSyncRequest(right), + ) => left == right, + ( + NetworkMessage::RbcDagShadowCarrierSyncResponse(left), + NetworkMessage::RbcDagShadowCarrierSyncResponse(right), + ) => left == right, + ( + NetworkMessage::RbcDagApplicationPayloadRequest(left), + NetworkMessage::RbcDagApplicationPayloadRequest(right), + ) => left == right, + ( + NetworkMessage::RbcDagApplicationPayloadResponse(left), + NetworkMessage::RbcDagApplicationPayloadResponse(right), + ) => { + left.application == right.application + && left.transaction_data.transactions() == right.transaction_data.transactions() + } + _ => false, + } +} + +fn current_timestamp_ns() -> TimestampNs { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .try_into() + .unwrap_or(TimestampNs::MAX) +} + +fn latency_since_timestamps( + creation_times: impl IntoIterator, + now_ns: TimestampNs, +) -> (u64, u64, u64) { + creation_times.into_iter().fold( + (0u64, 0u64, 0u64), + |(total, samples, maximum), creation_time| { + let latency = now_ns.saturating_sub(creation_time); + ( + total.saturating_add(latency), + samples.saturating_add(1), + maximum.max(latency), + ) + }, + ) +} + +#[derive(Default)] +struct CommitRoundDistanceBatch { + physical_forward: (u64, u64, u64), + physical_backward: (u64, u64, u64), +} + +impl CommitRoundDistanceBatch { + fn from_diagnostics( + diagnostics: impl IntoIterator, + ) -> Self { + let mut batch = Self::default(); + for diagnostic in diagnostics { + let aggregate = if diagnostic.physical_carrier_round_delta >= 0 { + &mut batch.physical_forward + } else { + &mut batch.physical_backward + }; + let value = diagnostic.physical_carrier_round_delta.unsigned_abs(); + aggregate.0 = aggregate.0.saturating_add(value); + aggregate.1 = aggregate.1.saturating_add(1); + aggregate.2 = aggregate.2.max(value); + } + batch + } + + fn observe(self, metrics: &Metrics) { + for (kind, (total, samples, maximum)) in [ + ( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, + self.physical_forward, + ), + ( + RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, + self.physical_backward, + ), + ] { + metrics.observe_starfish_rbc_dag_commit_round_distance(kind, total, samples, maximum); + } + } +} + /// Recover the exact locally selected Starfish-RBC chain so the persisted /// non-authoritative shadow can reconcile a WAL that ended before the direct /// DAG. The newest local block determines the branch when a Byzantine test @@ -147,10 +673,12 @@ fn recovered_local_rbc_headers( } fn shadow_transport_error_invalidates_run(error: &ShadowServiceErrorV1) -> bool { - matches!( - error, - ShadowServiceErrorV1::Overloaded { .. } | ShadowServiceErrorV1::Stopped - ) + match error { + ShadowServiceErrorV1::Stopped => true, + #[cfg(test)] + ShadowServiceErrorV1::Overloaded { .. } => true, + _ => false, + } } fn invalidate_shadow_run(metrics: &Metrics) { @@ -311,6 +839,49 @@ async fn send_network_message_reliably( } } +async fn run_rbc_dag_outbound_worker( + mailbox: RbcDagOutboundMailboxV1, + proactive_sender: mpsc::Sender, + priority_sender: mpsc::Sender, + mut outbound_failure: watch::Receiver>, +) -> Result<(), RbcDagOutboundMailboxErrorV1> { + loop { + if let Some(reason) = outbound_failure.borrow().clone() { + return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); + } + let next = tokio::select! { + biased; + changed = outbound_failure.changed() => { + if changed.is_err() { + return Err(RbcDagOutboundMailboxErrorV1::DownstreamClosed); + } + continue; + } + next = mailbox.recv() => next, + }; + let Some((class, message)) = next else { + let state = mailbox.inner.state.lock(); + return match &state.failure { + Some(reason) => Err(RbcDagOutboundMailboxErrorV1::Failed(reason.clone())), + None => Ok(()), + }; + }; + let result = match class { + RbcDagOutboundClassV1::Priority => priority_sender.try_send(message), + RbcDagOutboundClassV1::Proactive => proactive_sender.try_send(message), + }; + match result { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(_)) => { + return Err(RbcDagOutboundMailboxErrorV1::DownstreamSaturated(class)); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + return Err(RbcDagOutboundMailboxErrorV1::DownstreamClosed); + } + } + } +} + async fn broadcast_sailfish_cert_messages( senders: &[mpsc::Sender], cert_messages: &[crate::types::CertMessage], @@ -962,14 +1533,16 @@ impl ConnectionHandler ConnectionHandler { - self.handle_batch(*blocks).await; + if self.inner.embedded_rbc_authority { + tracing::warn!( + peer = self.peer_id, + "Rejected generic block batch while embedded RBC-DAG authority is active" + ); + } else { + self.handle_batch(*blocks).await; + } } NetworkMessage::MissingParentsRequest(refs) => { + if self.inner.embedded_rbc_authority { + tracing::debug!( + peer = self.peer_id, + count = refs.len(), + "Ignored legacy missing-parent request in standalone RBC-DAG mode" + ); + return true; + } return self.handle_missing_parents_request(refs).await; } NetworkMessage::MissingTxDataRequest(refs) => { + if self.inner.embedded_rbc_authority { + tracing::debug!( + peer = self.peer_id, + count = refs.len(), + "Ignored legacy transaction-data request in standalone RBC-DAG mode" + ); + return true; + } return self.handle_missing_tx_data_request(refs).await; } NetworkMessage::PartialSig(sig) => { @@ -1108,7 +1704,7 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier(self.peer_id, envelope) { + if let Err(error) = shadow.carrier_reliably(self.peer_id, envelope).await { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1118,7 +1714,10 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_request(self.peer_id, reference) { + if let Err(error) = shadow + .carrier_request_reliably(self.peer_id, reference) + .await + { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1128,7 +1727,10 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_response(self.peer_id, response) { + if let Err(error) = shadow + .carrier_response_reliably(self.peer_id, response) + .await + { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1138,7 +1740,10 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_sync_request(self.peer_id, request) { + if let Err(error) = shadow + .carrier_sync_request_reliably(self.peer_id, request) + .await + { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1150,7 +1755,10 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_sync_response(self.peer_id, response) { + if let Err(error) = shadow + .carrier_sync_response_reliably(self.peer_id, response) + .await + { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1160,6 +1768,39 @@ impl ConnectionHandler { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow + .application_payload_request_reliably(self.peer_id, application) + .await + { + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); + } + tracing::warn!( + ?application, + peer = self.peer_id, + "Failed to forward RBC-DAG application-payload request: {error}" + ); + } + } + } + NetworkMessage::RbcDagApplicationPayloadResponse(response) => { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow + .application_payload_response_reliably(self.peer_id, response) + .await + { + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); + } + tracing::warn!( + peer = self.peer_id, + "Failed to forward RBC-DAG application-payload response: {error}" + ); + } + } + } } true } @@ -1172,6 +1813,11 @@ impl ConnectionHandler { rbc_event_task: Option>, rbc_service_task: Option>, rbc_dag_shadow_event_task: Option>, + rbc_dag_core_control_task: Option>, + rbc_dag_assignment_task: Option>, rbc_dag_shadow_service_task: Option>, + rbc_dag_clock_bridge_state: Option>, cordial_knowledge_task: JoinHandle<()>, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RbcDagClockBridgeStateV1 { + Pending, + Failed, + Activated, +} + +type RbcDagPayloadVerificationTaskV1 = + JoinHandle, tokio::task::JoinError>>; + +enum RbcDagAuthorizedPayloadV1 { + None, + AlreadyAvailable(Option>), + Verify(RbcDagPayloadVerificationTaskV1), +} + +enum RbcDagCoreControlCommandV1 { + AuthorizedApplicationObserved { + carrier: BlockReference, + header: RbcCanonicalHeader, + authorization_basis: ShadowApplicationAuthorizationBasisV1, + payload: RbcDagAuthorizedPayloadV1, + }, + Frontier(CommittedFrontierDeltaV1), + /// Every recovery application and frontier preceding this marker must be + /// fully applied before activation is legal. + Ready, + Activate, + /// Intentional end-of-stream marker. A sender disappearing without this + /// marker is an authoritative runtime failure. + Drain, +} + +enum RbcDagApplicationAssignmentCommandV1 { + Assigned(BlockReference), + Drain, +} + +#[derive(Default)] +struct RbcDagCoreControlGateV1 { + startup_ready: bool, + clean_drain: bool, +} + +impl RbcDagCoreControlGateV1 { + fn record_ready(&mut self) { + self.startup_ready = true; + } + + fn activation_allowed(&self) -> bool { + self.startup_ready + } + + fn record_clean_drain(&mut self) { + self.clean_drain = true; + } +} + +async fn wait_for_rbc_dag_clock_bridge_activation( + bridge_state: &mut watch::Receiver, +) -> Result<(), String> { + loop { + match *bridge_state.borrow_and_update() { + RbcDagClockBridgeStateV1::Pending => {} + RbcDagClockBridgeStateV1::Failed => { + return Err( + "RBC-DAG startup recovery was invalid before Core clock activation".to_string(), + ); + } + RbcDagClockBridgeStateV1::Activated => return Ok(()), + } + bridge_state + .changed() + .await + .map_err(|_| "RBC-DAG event bridge stopped before Core clock activation".to_string())?; + } +} + +fn fail_rbc_dag_clock_bridge(bridge_tx: Option<&watch::Sender>) { + if let Some(bridge_tx) = bridge_tx { + // Failure is permanently terminal, including after a caller already + // observed activation. No later authority command may reach Core. + bridge_tx.send_if_modified(|state| { + if *state != RbcDagClockBridgeStateV1::Failed { + *state = RbcDagClockBridgeStateV1::Failed; + true + } else { + false + } + }); + } +} + +fn rbc_dag_clock_bridge_failed( + bridge_tx: Option<&watch::Sender>, +) -> bool { + bridge_tx.is_some_and(|bridge_tx| *bridge_tx.borrow() == RbcDagClockBridgeStateV1::Failed) +} + pub(crate) struct NetworkSyncSignals { block_ready_notify: Arc, proposal_round_notify: Arc, @@ -1773,11 +2521,21 @@ pub struct NetworkSyncerInner { /// another peer or the actor's local HeaderStaged/Delivered effects. rbc_peer_senders: parking_lot::RwLock>>, + /// Bounded, keyed RBC-DAG transport mailboxes. Exact repair is admitted + /// independently of proactive carrier fan-out and drains first. + rbc_dag_peer_mailboxes: parking_lot::RwLock>, pub leader_timeout: Duration, pub soft_block_timeout: Duration, + metrics: Arc, + rbc_dag_clock_bridge_tx: Option>, + rbc_dag_shutdown_started: Arc, /// Sailfish++ service handle for sending control messages /// (timeout/no-vote). None for non-SailfishPlusPlus protocols. pub sailfish_handle: Option, + /// When true, only typed carrier-authorized application ingress may reach + /// the core. Peer-controlled block batches and legacy data-recovery paths + /// are rejected even if their serialized `DataSource` claims authority. + pub embedded_rbc_authority: bool, /// Central Starfish-RBC service. Connection workers only forward their /// trusted peer identity and wire payload into this single owner. pub(crate) starfish_rbc_service: Option, @@ -1790,33 +2548,547 @@ pub struct NetworkSyncerInner { pub start_time: std::time::Instant, } -impl NetworkSyncer { - pub async fn start( - network: Network, - mut core: Core, - mut commit_observer: C, - metrics: Arc, - node_parameters: NodeParameters, - starfish_rbc_dag_shadow_wal: PathBuf, - partial_sig_outbox_rx: Option>, - bls_cert_aggregator: Option, - bls_signer: Option, - ) -> Self { - let handle = Handle::current(); - let block_ready_notify = Arc::new(Notify::new()); - let proposal_round_notify = Arc::new(Notify::new()); - let (committed, committed_leaders_count) = core.take_recovered_committed(); - commit_observer.recover_committed(committed, committed_leaders_count); - let committee = core.committee().clone(); - let mac_keys = core.mac_keys(); - let dag_state = core.dag_state().clone(); - let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { - match recovered_local_rbc_headers(&core) { - Ok(headers) => Some(headers), - Err(error) => { - // A partial local history would make delivery comparisons - // meaningless in mirror mode and make autonomous local - // application-origin reconciliation unsafe. Disable the +struct RbcDagAppliedFrontierObservationV1 { + carrier_count: u64, + application_count: u64, + application_creation_times: Vec, + commit_round_distances: CommitRoundDistanceBatch, +} + +trait RbcDagCoreControlTargetV1: Send + Sync { + async fn stage_authorized_application( + &self, + header: RbcCanonicalHeader, + authorization_basis: ShadowApplicationAuthorizationBasisV1, + ) -> Result<(), String>; + + fn restore_available_application( + &self, + application: BlockReference, + payload: Option>, + ) -> Result<(), String>; + + async fn materialize_authorized_payload( + &self, + item: ReconstructedTransactionData, + ) -> Result<(), String>; + + async fn apply_frontier(&self, delta: CommittedFrontierDeltaV1) -> Result; + + async fn activate_authority(&self) -> Result<(), String>; + + async fn apply_assignment(&self, reference: BlockReference) -> Result<(), String>; +} + +impl RbcDagCoreControlTargetV1 + for NetworkSyncerInner +{ + async fn stage_authorized_application( + &self, + header: RbcCanonicalHeader, + authorization_basis: ShadowApplicationAuthorizationBasisV1, + ) -> Result<(), String> { + let block_ref = header.reference(); + let (missing_parents, _) = self.syncer.add_authorized_rbc_dag_header(header).await; + self.cordial_knowledge + .send(CordialKnowledgeMessage::DagParts { + headers: vec![block_ref], + shards: Vec::new(), + }); + if !missing_parents.is_empty() { + tracing::debug!( + ?block_ref, + ?missing_parents, + ?authorization_basis, + "Authorized RBC-DAG application waits for parent materialization" + ); + } + Ok(()) + } + + fn restore_available_application( + &self, + application: BlockReference, + payload: Option>, + ) -> Result<(), String> { + // The actor is intentionally stopped before the authoritative FIFO is + // drained. Core already owns this data-available block, and restart + // recovery deterministically rehydrates the corresponding shadow + // state, so no actor callback is required while shutting down. + if self.rbc_dag_shutdown_started.load(Ordering::Acquire) { + return Ok(()); + } + let shadow = self + .starfish_rbc_dag_shadow_service + .as_ref() + .ok_or_else(|| "authoritative payload callback target is unavailable".to_owned())?; + if let Some(payload) = payload { + shadow + .verified_application_payload(application, payload) + .map_err(|error| error.to_string())?; + } + shadow + .application_data_available(application) + .map_err(|error| error.to_string()) + } + + async fn materialize_authorized_payload( + &self, + item: ReconstructedTransactionData, + ) -> Result<(), String> { + let block_ref = item.block_reference; + self.cordial_knowledge + .send(CordialKnowledgeMessage::DagParts { + headers: Vec::new(), + shards: vec![block_ref], + }); + if let Some(shard_tx) = self.shard_tx.lock().as_ref() { + let _ = shard_tx.send(vec![ShardMessage::FullBlock(block_ref)]); + } + let verified_payload = Arc::new(item.transaction_data.clone()); + self.syncer.add_authorized_rbc_dag_payload(item).await; + // Core materialization is durable authority work and must complete + // before a queued frontier. The actor callback is ephemeral and the + // actor has already been stopped by the graceful shutdown protocol. + if self.rbc_dag_shutdown_started.load(Ordering::Acquire) { + return Ok(()); + } + self.starfish_rbc_dag_shadow_service + .as_ref() + .ok_or_else(|| "authoritative payload callback target is unavailable".to_owned())? + .verified_application_payload(block_ref, verified_payload) + .map_err(|error| error.to_string()) + } + + async fn apply_frontier(&self, delta: CommittedFrontierDeltaV1) -> Result { + self.syncer + .apply_starfish_rbc_dag_frontier(delta) + .await + .map_err(|error| error.to_string()) + } + + async fn activate_authority(&self) -> Result<(), String> { + self.syncer.activate_starfish_rbc_dag_authority().await; + Ok(()) + } + + async fn apply_assignment(&self, reference: BlockReference) -> Result<(), String> { + self.syncer + .apply_starfish_rbc_dag_application_assigned(reference) + .await; + Ok(()) + } +} + +impl RbcDagAppliedFrontierObservationV1 { + fn from_delta(delta: &CommittedFrontierDeltaV1) -> Self { + Self { + carrier_count: delta.carriers.len() as u64, + application_count: delta.applications.len() as u64, + application_creation_times: delta + .applications + .iter() + .map(RbcCanonicalHeader::meta_creation_time_ns) + .collect(), + commit_round_distances: CommitRoundDistanceBatch::from_diagnostics( + delta.application_diagnostics.iter().copied(), + ), + } + } + + fn observe(self, metrics: &Metrics) { + let (total_ns, samples, max_ns) = + latency_since_timestamps(self.application_creation_times, current_timestamp_ns()); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, + total_ns, + samples, + max_ns, + ); + self.commit_round_distances.observe(metrics); + metrics.starfish_rbc_dag_frontier_applied(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .inc(); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "carrier"]) + .inc_by(self.carrier_count); + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .inc_by(self.application_count); + } +} + +fn fail_rbc_dag_authority( + metrics: &Metrics, + bridge_tx: Option<&watch::Sender>, + reason: &str, +) { + invalidate_shadow_run(metrics); + fail_rbc_dag_clock_bridge(bridge_tx); + tracing::error!( + reason, + "RBC-DAG authoritative Core-control worker failed closed" + ); +} + +fn fail_rbc_dag_outbound_transport( + metrics: &Metrics, + bridge_tx: Option<&watch::Sender>, + embedded_rbc_authority: bool, + recipient: AuthorityIndex, + error: &RbcDagOutboundMailboxErrorV1, +) { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "overloaded"]) + .inc(); + if embedded_rbc_authority { + fail_rbc_dag_authority( + metrics, + bridge_tx, + &format!("RBC-DAG outbound mailbox for authority {recipient} failed: {error}"), + ); + } else { + invalidate_shadow_run(metrics); + tracing::error!( + peer = recipient, + ?error, + "RBC-DAG observational outbound mailbox failed" + ); + } +} + +struct RbcDagEventRouterGuardV1 { + metrics: Arc, + bridge_tx: watch::Sender, + clean_exit: bool, +} + +impl RbcDagEventRouterGuardV1 { + fn new(metrics: Arc, bridge_tx: watch::Sender) -> Self { + Self { + metrics, + bridge_tx, + clean_exit: false, + } + } + + fn record_clean_exit(&mut self) { + self.clean_exit = true; + } +} + +impl Drop for RbcDagEventRouterGuardV1 { + fn drop(&mut self) { + if !self.clean_exit { + fail_rbc_dag_authority( + &self.metrics, + Some(&self.bridge_tx), + "RBC-DAG authoritative event router stopped unexpectedly", + ); + } + } +} + +async fn drain_rbc_dag_authorized_payload(payload: RbcDagAuthorizedPayloadV1) { + if let RbcDagAuthorizedPayloadV1::Verify(payload_verification) = payload { + // Await rather than merely drop the JoinHandle: dropping detaches the + // verifier and could let Reed-Solomon work outlive Core shutdown. + let _ = payload_verification.await; + } +} + +async fn drain_rbc_dag_control_command_payload(command: RbcDagCoreControlCommandV1) { + if let RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { payload, .. } = command { + drain_rbc_dag_authorized_payload(payload).await; + } +} + +async fn enqueue_rbc_dag_core_control( + sender: &mpsc::Sender, + command: RbcDagCoreControlCommandV1, + metrics: &Metrics, + bridge_tx: Option<&watch::Sender>, +) -> bool { + let drain = matches!(command, RbcDagCoreControlCommandV1::Drain); + if !drain && rbc_dag_clock_bridge_failed(bridge_tx) { + drain_rbc_dag_control_command_payload(command).await; + return false; + } + match sender.send(command).await { + Ok(()) => return true, + Err(error) => drain_rbc_dag_control_command_payload(error.0).await, + } + fail_rbc_dag_authority( + metrics, + bridge_tx, + "bounded RBC-DAG Core-control channel closed unexpectedly", + ); + false +} + +async fn run_rbc_dag_core_control_worker( + target: Arc, + metrics: Arc, + bridge_tx: watch::Sender, + shutdown_started: Arc, + mut commands: mpsc::Receiver, +) { + let mut gate = RbcDagCoreControlGateV1::default(); + while let Some(command) = commands.recv().await { + if matches!(command, RbcDagCoreControlCommandV1::Drain) { + gate.record_clean_drain(); + break; + } + if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { + drain_rbc_dag_control_command_payload(command).await; + continue; + } + if !shutdown_started.load(Ordering::Acquire) + && metrics.starfish_rbc_dag_shadow_clock_valid.get() == 0 + { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + "authoritative runtime validity was revoked", + ); + drain_rbc_dag_control_command_payload(command).await; + continue; + } + + match command { + RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier, + header, + authorization_basis, + payload, + } => { + let block_ref = header.reference(); + if let Err(error) = target + .stage_authorized_application(header, authorization_basis) + .await + { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!("authorized header insertion failed for {block_ref}: {error}"), + ); + drain_rbc_dag_authorized_payload(payload).await; + continue; + } + + let item = match payload { + RbcDagAuthorizedPayloadV1::None => continue, + RbcDagAuthorizedPayloadV1::AlreadyAvailable(payload) => { + if !shutdown_started.load(Ordering::Acquire) { + if let Err(error) = + target.restore_available_application(block_ref, payload) + { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!( + "materialized payload callback failed for {block_ref}: {error}" + ), + ); + } + } + continue; + } + RbcDagAuthorizedPayloadV1::Verify(payload_verification) => { + match payload_verification.await { + Ok(Ok(Ok(item))) => item, + // Payload bytes remain untrusted even when their + // enclosing header is authorized. Reject only this + // observation so another holder can recover it. + Ok(Ok(Err(error))) => { + tracing::warn!( + ?carrier, + ?block_ref, + ?authorization_basis, + ?error, + "Rejected RBC-DAG application payload" + ); + continue; + } + Ok(Err(error)) | Err(error) + if shutdown_started.load(Ordering::Acquire) => + { + tracing::debug!( + ?block_ref, + ?error, + "RBC-DAG payload verifier stopped during shutdown" + ); + continue; + } + Ok(Err(error)) | Err(error) => { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!( + "authorized payload verifier stopped for {block_ref} in carrier {carrier}: {error}" + ), + ); + continue; + } + } + } + }; + if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { + continue; + } + if let Err(error) = target.materialize_authorized_payload(item).await { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!("verified payload callback failed for {block_ref}: {error}"), + ); + } + } + RbcDagCoreControlCommandV1::Frontier(delta) => { + let observation = RbcDagAppliedFrontierObservationV1::from_delta(&delta); + match target.apply_frontier(delta).await { + Ok(true) => observation.observe(&metrics), + // Exact replay is an idempotent Core acknowledgment. It + // must not double-count application or frontier effects. + Ok(false) => {} + Err(error) => fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!("authoritative frontier was rejected: {error}"), + ), + } + } + RbcDagCoreControlCommandV1::Ready => gate.record_ready(), + RbcDagCoreControlCommandV1::Activate => { + if !gate.activation_allowed() { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + "clock activation arrived before the startup recovery barrier", + ); + continue; + } + if let Err(error) = target.activate_authority().await { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!("Core authority activation failed: {error}"), + ); + continue; + } + if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { + continue; + } + metrics.starfish_rbc_dag_shadow_clock_valid.set(1); + bridge_tx.send_if_modified(|state| { + if *state == RbcDagClockBridgeStateV1::Pending { + *state = RbcDagClockBridgeStateV1::Activated; + true + } else { + false + } + }); + } + RbcDagCoreControlCommandV1::Drain => unreachable!("drain handled above"), + } + } + + if !gate.clean_drain { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + "RBC-DAG Core-control sender disappeared before a graceful drain", + ); + } +} + +async fn run_rbc_dag_application_assignment_worker( + target: Arc, + metrics: Arc, + bridge_tx: watch::Sender, + shutdown_started: Arc, + mut assignments: mpsc::Receiver, +) { + let mut clean_drain = false; + while let Some(command) = assignments.recv().await { + match command { + RbcDagApplicationAssignmentCommandV1::Drain => { + clean_drain = true; + break; + } + RbcDagApplicationAssignmentCommandV1::Assigned(reference) => { + // Assignments release an ephemeral producer gate. The local + // application was already durably fixed before this event, so + // creating a fresh Core block while the shadow actor is + // stopped is both unnecessary and unsafe. + if shutdown_started.load(Ordering::Acquire) { + continue; + } + if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { + continue; + } + if let Err(error) = target.apply_assignment(reference).await { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + &format!("Core application-assignment acknowledgment failed: {error}"), + ); + } + } + } + } + if !clean_drain && !shutdown_started.load(Ordering::Acquire) { + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + "RBC-DAG assignment sender disappeared unexpectedly", + ); + } +} + +impl NetworkSyncer { + pub async fn start( + network: Network, + mut core: Core, + mut commit_observer: C, + metrics: Arc, + node_parameters: NodeParameters, + starfish_rbc_dag_shadow_wal: PathBuf, + partial_sig_outbox_rx: Option>, + bls_cert_aggregator: Option, + bls_signer: Option, + rbc_dag_clock_start_paused: bool, + ) -> Self { + let handle = Handle::current(); + let block_ready_notify = Arc::new(Notify::new()); + let proposal_round_notify = Arc::new(Notify::new()); + let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; + let (committed, committed_leaders_count) = + core.take_recovered_committed(embedded_rbc_authority); + commit_observer.recover_committed(committed, committed_leaders_count); + let committee = core.committee().clone(); + let mac_keys = core.mac_keys(); + let dag_state = core.dag_state().clone(); + let rbc_dag_frontier_recovery_cursor = embedded_rbc_authority + .then(|| core.rbc_dag_frontier_recovery_cursor()) + .flatten(); + let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { + match recovered_local_rbc_headers(&core) { + Ok(headers) => Some(headers), + Err(error) if embedded_rbc_authority => { + panic!( + "embedded RBC-DAG authority cannot start without reconcilable local history: {error}" + ); + } + Err(error) => { + // A partial local history would make delivery comparisons + // meaningless in mirror mode and make autonomous local + // application-origin reconciliation unsafe. Disable the // RBC-DAG runtime while allowing the legacy path to continue. tracing::error!( "Disabling RBC-DAG runtime because recovered direct headers cannot be reconciled: {error}" @@ -1857,7 +3129,7 @@ impl NetworkSyncer .as_ref() .map(|tx| SailfishServiceHandle::new(tx.clone())); let (starfish_rbc_service, rbc_event_rx, rbc_service_task) = - if dag_state.consensus_protocol.is_starfish_rbc() { + if dag_state.consensus_protocol.is_starfish_rbc() && !embedded_rbc_authority { let protocol_instance = node_parameters .starfish_rbc_protocol_instance .and_then(|bytes| RbcProtocolInstanceId::new(bytes).ok()) @@ -1876,11 +3148,6 @@ impl NetworkSyncer } BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, }; - let phase_authority = if node_parameters.starfish_rbc_dag_embedded_rbc_authority { - RbcPhaseAuthorityV1::EmbeddedCarrierDag - } else { - RbcPhaseAuthorityV1::Direct - }; let (service, events, task) = start_starfish_rbc_service_with_phase_authority( committee.clone(), dag_state.get_own_authority_index(), @@ -1890,7 +3157,7 @@ impl NetworkSyncer initial_authenticator, dag_state.highest_round(), STARFISH_RBC_HEADER_RETRY_INTERVAL, - phase_authority, + RbcPhaseAuthorityV1::Direct, ) .expect("validated Starfish-RBC configuration must start its service"); (Some(service), Some(events), Some(task)) @@ -1943,26 +3210,48 @@ impl NetworkSyncer } else { ShadowWalSyncPolicyV1::EveryBatch }; - start_starfish_rbc_dag_autonomous_clock_service_v1( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - recovered_local_headers, - // The idle carrier pacemaker deliberately shares the - // resolved Starfish leader timeout. Application and - // embedded RBC phase carriers remain event-driven. - node_parameters.leader_timeout, - wal_sync_policy, - ) + if embedded_rbc_authority { + start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + recovered_local_headers, + // The idle carrier pacemaker deliberately shares the + // resolved Starfish leader timeout. Application and + // embedded RBC phase carriers remain event-driven. + node_parameters.leader_timeout, + wal_sync_policy, + Arc::clone(&metrics), + rbc_dag_frontier_recovery_cursor, + !rbc_dag_clock_start_paused, + ) + } else { + let start = if rbc_dag_clock_start_paused { + start_starfish_rbc_dag_autonomous_clock_service_paused_with_metrics_v1 + } else { + start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1 + }; + start( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + recovered_local_headers, + node_parameters.leader_timeout, + wal_sync_policy, + Arc::clone(&metrics), + ) + } } else { let wal_sync_policy = if node_parameters.starfish_rbc_dag_shadow_buffered_wal { ShadowWalSyncPolicyV1::OnShutdown } else { ShadowWalSyncPolicyV1::EveryBatch }; - start_starfish_rbc_dag_shadow_service_v1( + start_starfish_rbc_dag_shadow_service_with_metrics_v1( starfish_rbc_dag_shadow_wal, committee_context, dag_state.get_own_authority_index(), @@ -1970,10 +3259,14 @@ impl NetworkSyncer authorizer, recovered_local_headers, wal_sync_policy, + Arc::clone(&metrics), ) }; match started { Ok((service, events, task)) => (Some(service), Some(events), Some(task)), + Err(error) if embedded_rbc_authority => { + panic!("embedded RBC-DAG authority failed to start: {error}"); + } Err(error) => { invalidate_shadow_run(&metrics); tracing::error!("Disabling Starfish-RBC-DAG runtime: {error}"); @@ -1983,7 +3276,6 @@ impl NetworkSyncer } else { (None, None, None) }; - let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; let syncer = Syncer::new( core, NetworkSyncSignals { @@ -2000,10 +3292,11 @@ impl NetworkSyncer ); let initial_round = syncer.core().next_block_round(); let syncer = CoreThreadDispatcher::start(syncer); - // Await the initial command while the async RBC actor remains - // schedulable. The command itself runs on the dedicated core thread, - // where synchronous local-INIT selection is safe. - syncer.force_new_block(initial_round).await; + if !embedded_rbc_authority { + // Embedded authority production is released by the ordered Ready + // bridge only after replayed frontier receipts are durable. + syncer.force_new_block(initial_round).await; + } let (stop_sender, stop_receiver) = mpsc::channel(1); // Occupy the only available permit, so that all other // calls to send() will block. @@ -2047,6 +3340,17 @@ impl NetworkSyncer dag_state.attach_cordial_knowledge(cordial_knowledge_handle.clone()); let cordial_knowledge_task = handle.spawn(cordial_knowledge_actor.run()); + let (rbc_dag_clock_bridge_tx, rbc_dag_clock_bridge_state) = if node_parameters + .starfish_rbc_dag_autonomous_clock + && starfish_rbc_dag_shadow_service.is_some() + { + let (tx, rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + let rbc_dag_shutdown_started = Arc::new(AtomicBool::new(false)); + let inner = Arc::new(NetworkSyncerInner { block_ready_notify, dag_state: dag_state.clone(), @@ -2062,9 +3366,14 @@ impl NetworkSyncer cordial_knowledge: cordial_knowledge_handle, peer_senders: parking_lot::RwLock::new(AHashMap::new()), rbc_peer_senders: parking_lot::RwLock::new(AHashMap::new()), + rbc_dag_peer_mailboxes: parking_lot::RwLock::new(AHashMap::new()), leader_timeout: node_parameters.leader_timeout, soft_block_timeout: node_parameters.soft_block_timeout, + metrics: metrics.clone(), + rbc_dag_clock_bridge_tx: rbc_dag_clock_bridge_tx.clone(), + rbc_dag_shutdown_started: rbc_dag_shutdown_started.clone(), sailfish_handle: sf_handle_for_inner, + embedded_rbc_authority, starfish_rbc_service: starfish_rbc_service.clone(), starfish_rbc_dag_shadow_service: starfish_rbc_dag_shadow_service.clone(), start_time: std::time::Instant::now(), @@ -2166,17 +3475,6 @@ impl NetworkSyncer .syncer .add_transaction_data(vec![item], DataSource::StarfishRbcPayload) .await; - if let Some(ref shadow) = - event_inner.starfish_rbc_dag_shadow_service - { - if let Err(error) = shadow.application_data_available(block_ref) { - invalidate_shadow_run(&rbc_metrics); - tracing::warn!( - ?block_ref, - "Failed to record RBC-DAG application availability: {error}" - ); - } - } } RbcServiceEvent::Delivered(header) => { if let Some(ref shadow) = @@ -2218,43 +3516,129 @@ impl NetworkSyncer RbcCommitteeId::derive(&inner.committee) .expect("validated direct RBC committee must retain a stable identifier") }); + let (rbc_dag_core_control_tx, rbc_dag_core_control_task) = if embedded_rbc_authority { + let (control_tx, control_rx) = mpsc::channel(STARFISH_RBC_DAG_CORE_CONTROL_CAPACITY); + let worker_inner = inner.clone(); + let worker_metrics = metrics.clone(); + let worker_bridge_tx = rbc_dag_clock_bridge_tx + .as_ref() + .expect("embedded authority must supervise clock activation") + .clone(); + let panic_metrics = metrics.clone(); + let panic_bridge_tx = worker_bridge_tx.clone(); + let worker_shutdown_started = rbc_dag_shutdown_started.clone(); + let task = handle.spawn(async move { + if AssertUnwindSafe(run_rbc_dag_core_control_worker( + worker_inner, + worker_metrics, + worker_bridge_tx, + worker_shutdown_started, + control_rx, + )) + .catch_unwind() + .await + .is_err() + { + fail_rbc_dag_authority( + &panic_metrics, + Some(&panic_bridge_tx), + "RBC-DAG Core-control worker panicked", + ); + } + }); + (Some(control_tx), Some(task)) + } else { + (None, None) + }; + let (rbc_dag_assignment_tx, rbc_dag_assignment_task) = if embedded_rbc_authority { + // Capacity one plus the single in-flight dispatcher call preserves + // assignment order with a strict two-item bound, independently of + // remote payload verification in the authority FIFO. + let (assignment_tx, assignment_rx) = mpsc::channel(1); + let assignment_inner = inner.clone(); + let assignment_metrics = metrics.clone(); + let assignment_bridge_tx = rbc_dag_clock_bridge_tx + .as_ref() + .expect("embedded authority must supervise assignments") + .clone(); + let assignment_shutdown_started = rbc_dag_shutdown_started.clone(); + let panic_metrics = metrics.clone(); + let panic_bridge_tx = assignment_bridge_tx.clone(); + let task = handle.spawn(async move { + if AssertUnwindSafe(run_rbc_dag_application_assignment_worker( + assignment_inner, + assignment_metrics, + assignment_bridge_tx, + assignment_shutdown_started, + assignment_rx, + )) + .catch_unwind() + .await + .is_err() + { + fail_rbc_dag_authority( + &panic_metrics, + Some(&panic_bridge_tx), + "RBC-DAG assignment worker panicked", + ); + } + }); + (Some(assignment_tx), Some(task)) + } else { + (None, None) + }; let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); let shadow_metrics = metrics.clone(); + let rbc_dag_clock_bridge_tx = rbc_dag_clock_bridge_tx.clone(); + let rbc_dag_core_control_tx = rbc_dag_core_control_tx; + let rbc_dag_assignment_tx = rbc_dag_assignment_tx; + let rbc_dag_shutdown_started = rbc_dag_shutdown_started; handle.spawn(async move { + let mut router_guard = embedded_rbc_authority.then(|| { + RbcDagEventRouterGuardV1::new( + shadow_metrics.clone(), + rbc_dag_clock_bridge_tx + .as_ref() + .expect("embedded authority must supervise its event router") + .clone(), + ) + }); + // Keep expensive Reed-Solomon work off the carrier event + // router. Verification remains parallel, while the bounded + // Core-control worker awaits its handles in exact service + // order before applying a later frontier or activation. + let payload_verification_limit = Arc::new(Semaphore::new(4)); while let Some(event) = event_rx.recv().await { match event { ShadowServiceEventV1::Network { recipient, message } => { - let sender = event_inner.peer_senders.read().get(&recipient).cloned(); - if let Some(sender) = sender { - match sender.try_send(message) { - Ok(()) => shadow_metrics + // The per-peer keyed mailbox bounds both proactive + // history and exact-repair traffic. Distinct-key + // saturation is a transport failure: no proactive + // item is silently evicted to make room for a + // newer one. + let mailbox = event_inner + .rbc_dag_peer_mailboxes + .read() + .get(&recipient) + .cloned(); + if let Some(mailbox) = mailbox { + match mailbox.enqueue(message, &event_inner.committee) { + Ok(RbcDagOutboundEnqueueV1::Added) => shadow_metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["network", "sent"]) .inc(), - Err(mpsc::error::TrySendError::Full(_)) => { - // The shadow is observational. It must - // shed work instead of backpressuring - // the authoritative network path. - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "dropped_backpressure"]) - .inc(); - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "disconnected"]) - .inc(); - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - } + Ok(RbcDagOutboundEnqueueV1::Coalesced) => shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "coalesced"]) + .inc(), + Err(error) => fail_rbc_dag_outbound_transport( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + embedded_rbc_authority, + recipient, + &error, + ), } } else { shadow_metrics @@ -2292,34 +3676,181 @@ impl NetworkSyncer ) { Ok(_) => {} Err(error) => { - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - tracing::error!( - ?carrier, - ?error, - "Embedded RBC delivered an invalid application header" + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + &format!( + "embedded carrier {carrier:?} delivered an invalid application header: {error}" + ), ); } } } } - ShadowServiceEventV1::FrontierCommitted(delta) => { - if embedded_rbc_authority { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .inc(); - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "carrier"]) - .inc_by(delta.carriers.len() as u64); - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .inc_by(delta.applications.len() as u64); - event_inner - .syncer - .apply_starfish_rbc_dag_frontier(delta) - .await; + ShadowServiceEventV1::AuthorizedApplicationObserved { + carrier, + header, + payload, + authorization_basis, + } => { + if !embedded_rbc_authority { + tracing::debug!( + ?carrier, + ?authorization_basis, + "Ignoring standalone application event outside embedded authority mode" + ); + continue; + } + // Once authority has failed, do not even start a + // new payload verifier. Commands already accepted + // by the bounded FIFO are joined by its worker. + if rbc_dag_clock_bridge_failed(rbc_dag_clock_bridge_tx.as_ref()) { + continue; + } + let pinned = match PinnedRbcHeader::validate_with_committee_id( + header, + &event_inner.committee, + embedded_rbc_committee_id + .expect("embedded authority must cache its committee ID"), + ) { + Ok(pinned) => pinned, + Err(error) => { + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + &format!( + "carrier {carrier:?} emitted an invalid authorized application header ({authorization_basis:?}): {error}" + ), + ); + continue; + } + }; + let canonical = pinned.header().clone(); + let block_ref = canonical.reference(); + let control_payload = if event_inner + .dag_state + .is_data_available(&block_ref) + { + let Some(block) = + event_inner.dag_state.get_storage_block(block_ref) + else { + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + &format!( + "data-available application {block_ref} has no storage block" + ), + ); + continue; + }; + let transaction_data = block.transaction_data().cloned(); + if transaction_data.is_none() + && canonical.transactions_commitment() + != TransactionsCommitment::default() + { + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + &format!( + "data-available application {block_ref} has no transaction payload" + ), + ); + continue; + } + // Empty committed transaction sets are + // data-available without a TransactionData + // allocation. The ordered availability callback + // remains required, but there is no payload to + // verify or cache. + RbcDagAuthorizedPayloadV1::AlreadyAvailable( + transaction_data.map(Arc::new), + ) + } else if let Some(transaction_data) = payload { + let payload_verification_limit = + payload_verification_limit.clone(); + let committee = event_inner.committee.clone(); + let mac_keys = event_inner.mac_keys.clone(); + let own_id = event_inner.dag_state.get_own_authority_index(); + let authentication_scheme = + event_inner.dag_state.block_authentication_scheme; + let verification_header = canonical.clone(); + let task = tokio::spawn(async move { + let permit = payload_verification_limit + .acquire_owned() + .await + .expect("RBC-DAG payload semaphore remains open"); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let mut encoder = ReedSolomonEncoder::new(2, 4, 2) + .expect("RBC-DAG payload encoder should be created"); + verify_starfish_rbc_transaction_payload( + &verification_header, + transaction_data, + &committee, + own_id, + block_ref.authority, + &mut encoder, + authentication_scheme, + &mac_keys, + ) + }) + .await + }); + RbcDagAuthorizedPayloadV1::Verify(task) + } else { + RbcDagAuthorizedPayloadV1::None + }; + if let Some(ref control_tx) = rbc_dag_core_control_tx { + enqueue_rbc_dag_core_control( + control_tx, + RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier, + header: canonical, + authorization_basis, + payload: control_payload, + }, + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + ) + .await; + } + } + ShadowServiceEventV1::ApplicationAssigned(reference) => { + if let Some(ref assignment_tx) = rbc_dag_assignment_tx { + if rbc_dag_clock_bridge_failed(rbc_dag_clock_bridge_tx.as_ref()) { + continue; + } + // Assignment releases the one-outstanding + // producer gate and has no frontier/recovery + // dependency. Keep it off the payload-heavy + // FIFO; capacity one preserves exact order + // without spawning unbounded tasks. + if assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Assigned( + reference, + )) + .await + .is_err() + { + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + "bounded RBC-DAG assignment channel closed unexpectedly", + ); + } + } + } + ShadowServiceEventV1::FrontierCommitted(delta) => { + if let Some(ref control_tx) = rbc_dag_core_control_tx { + enqueue_rbc_dag_core_control( + control_tx, + RbcDagCoreControlCommandV1::Frontier(delta), + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + ) + .await; + } else { + shadow_metrics.starfish_rbc_dag_frontier_ignored(); } } ShadowServiceEventV1::VertexProjected(reference) => { @@ -2407,13 +3938,43 @@ impl NetworkSyncer } } ShadowServiceEventV1::Ready { autonomous_clock } => { - let verdict = if autonomous_clock { - &shadow_metrics.starfish_rbc_dag_shadow_clock_valid + if autonomous_clock && embedded_rbc_authority { + if let Some(ref control_tx) = rbc_dag_core_control_tx { + // FIFO placement makes this a barrier over + // every recovered header, payload, and + // committed frontier emitted before Ready. + enqueue_rbc_dag_core_control( + control_tx, + RbcDagCoreControlCommandV1::Ready, + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + ) + .await; + } } else { - &shadow_metrics.starfish_rbc_dag_shadow_comparison_valid - }; - if verdict.get() != 0 { - verdict.set(1); + let verdict = + &shadow_metrics.starfish_rbc_dag_shadow_comparison_valid; + if verdict.get() != 0 { + verdict.set(1); + } + } + } + ShadowServiceEventV1::ClockActivated => { + if let Some(ref control_tx) = rbc_dag_core_control_tx { + enqueue_rbc_dag_core_control( + control_tx, + RbcDagCoreControlCommandV1::Activate, + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + ) + .await; + } else if shadow_metrics.starfish_rbc_dag_shadow_clock_valid.get() == 0 { + fail_rbc_dag_clock_bridge(rbc_dag_clock_bridge_tx.as_ref()); + } else { + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(1); + if let Some(ref bridge_tx) = rbc_dag_clock_bridge_tx { + bridge_tx.send_replace(RbcDagClockBridgeStateV1::Activated); + } } } ShadowServiceEventV1::ClockState { @@ -2457,10 +4018,17 @@ impl NetworkSyncer } ShadowServiceEventV1::Rejected { peer, error } => { if peer.is_none() { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + if embedded_rbc_authority { + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + &format!( + "RBC-DAG actor rejected authoritative runtime input: {error}" + ), + ); + } else { + invalidate_shadow_run(&shadow_metrics); + } } tracing::warn!( "Rejected RBC-DAG runtime input from {:?}: {}", @@ -2470,14 +4038,39 @@ impl NetworkSyncer } } } + if rbc_dag_shutdown_started.load(Ordering::Acquire) { + if let Some(ref assignment_tx) = rbc_dag_assignment_tx { + let _ = assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Drain) + .await; + } + } + if let Some(ref control_tx) = rbc_dag_core_control_tx { + if rbc_dag_shutdown_started.load(Ordering::Acquire) { + enqueue_rbc_dag_core_control( + control_tx, + RbcDagCoreControlCommandV1::Drain, + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + ) + .await; + } else { + fail_rbc_dag_authority( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + "RBC-DAG actor event stream closed unexpectedly", + ); + } + } + if let Some(ref mut router_guard) = router_guard { + router_guard.record_clean_exit(); + } }) }); // Start bridge task that forwards reconstructed transaction data to core - let bridge_metrics = metrics.clone(); let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); - let bridge_metrics = bridge_metrics.clone(); handle.spawn(async move { while let Some(items) = decoded_rx.recv().await { // Reconstruction proves we now have the shard data for the @@ -2496,17 +4089,6 @@ impl NetworkSyncer .syncer .add_transaction_data(items, DataSource::ShardReconstructor) .await; - if let Some(ref shadow) = bridge_inner.starfish_rbc_dag_shadow_service { - for reference in shard_refs { - if let Err(error) = shadow.application_data_available(reference) { - invalidate_shadow_run(&bridge_metrics); - tracing::warn!( - ?reference, - "Failed to record reconstructed RBC-DAG availability: {error}" - ); - } - } - } } }) }); @@ -2775,15 +4357,170 @@ impl NetworkSyncer rbc_event_task, rbc_service_task, rbc_dag_shadow_event_task, + rbc_dag_core_control_task, + rbc_dag_assignment_task, rbc_dag_shadow_service_task, + rbc_dag_clock_bridge_state, cordial_knowledge_task, } } - pub(crate) async fn shutdown(self) -> Syncer { + pub(crate) async fn activate_starfish_rbc_dag_clock(&self) -> Result<(), String> { + let shadow = self + .inner + .starfish_rbc_dag_shadow_service + .clone() + .ok_or_else(|| "RBC-DAG clock service is not running".to_string())?; + let mut bridge_state = self + .rbc_dag_clock_bridge_state + .clone() + .ok_or_else(|| "RBC-DAG autonomous event bridge is not running".to_string())?; + match *bridge_state.borrow() { + RbcDagClockBridgeStateV1::Pending => {} + RbcDagClockBridgeStateV1::Failed => { + return Err( + "RBC-DAG startup recovery was invalid before clock activation".to_string(), + ); + } + RbcDagClockBridgeStateV1::Activated => return Ok(()), + } + + shadow + .activate_clock() + .await + .map_err(|error| format!("RBC-DAG clock activation failed: {error}"))?; + wait_for_rbc_dag_clock_bridge_activation(&mut bridge_state).await + } + + pub(crate) async fn shutdown(self) -> Option> { drop(self.stop); - // todo - wait for network shutdown as well + // Stop new network/main ingress before asking the authoritative actor + // to close. The actor, ordered router, bounded workers, and Core must + // all remain alive until their already accepted work is drained. self.main_task.await.ok(); + self.inner + .rbc_dag_shutdown_started + .store(true, Ordering::Release); + + let mut shadow_shutdown_timed_out = false; + if let Some(ref shadow) = self.inner.starfish_rbc_dag_shadow_service { + let shutdown_timeout = if self.inner.embedded_rbc_authority { + STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT + } else { + STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT + }; + match tokio::time::timeout(shutdown_timeout, shadow.shutdown()).await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!("RBC-DAG runtime did not acknowledge shutdown: {error}"); + if self.inner.embedded_rbc_authority { + fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + &format!("authoritative actor shutdown failed: {error}"), + ); + } + } + Err(_) => { + shadow_shutdown_timed_out = true; + tracing::warn!("Timed out stopping RBC-DAG runtime"); + if self.inner.embedded_rbc_authority { + fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + "authoritative actor shutdown timed out", + ); + } + } + } + } + + let mut rbc_dag_shadow_service_task = self.rbc_dag_shadow_service_task; + if let Some(mut actor_task) = rbc_dag_shadow_service_task.take() { + if shadow_shutdown_timed_out { + actor_task.abort(); + actor_task.await.ok(); + } else { + match tokio::time::timeout(STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT, &mut actor_task) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + if self.inner.embedded_rbc_authority { + fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + &format!( + "authoritative RBC-DAG actor supervisor failed during shutdown: {error}" + ), + ); + } else { + tracing::warn!( + "RBC-DAG actor supervisor failed during shutdown: {error}" + ); + } + } + Err(_) => { + fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + "RBC-DAG actor supervisor did not stop after shutdown", + ); + actor_task.abort(); + actor_task.await.ok(); + } + } + } + } + + if let Some(mut router_task) = self.rbc_dag_shadow_event_task { + match tokio::time::timeout(STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT, &mut router_task) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + &format!("RBC-DAG event router failed during shutdown: {error}"), + ), + Err(_) => { + fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + "RBC-DAG event router drain timed out", + ); + router_task.abort(); + router_task.await.ok(); + } + } + } + + for (name, task) in [ + ("assignment", self.rbc_dag_assignment_task), + ("Core-control", self.rbc_dag_core_control_task), + ] { + let Some(mut task) = task else { + continue; + }; + match tokio::time::timeout(STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT, &mut task).await { + Ok(Ok(())) => {} + Ok(Err(error)) => fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + &format!("RBC-DAG {name} worker failed during shutdown: {error}"), + ), + Err(_) => { + fail_rbc_dag_authority( + &self.inner.metrics, + self.inner.rbc_dag_clock_bridge_tx.as_ref(), + &format!("RBC-DAG {name} worker drain timed out"), + ); + task.abort(); + task.await.ok(); + } + } + } + // Close the shard reconstructor channel so the bridge task can exit // and release its Arc reference. self.inner.shard_tx.lock().take(); @@ -2821,11 +4558,6 @@ impl NetworkSyncer rbc_task.await.ok(); } let rbc_service_task = self.rbc_service_task; - if let Some(shadow_task) = self.rbc_dag_shadow_event_task { - shadow_task.abort(); - shadow_task.await.ok(); - } - let rbc_dag_shadow_service_task = self.rbc_dag_shadow_service_task; // Stop the cordial knowledge actor. self.cordial_knowledge_task.abort(); self.cordial_knowledge_task.await.ok(); @@ -2834,63 +4566,49 @@ impl NetworkSyncer // observe `stopped()`. Give them a short window to drop their `Arc`s // before insisting on `try_unwrap`. let mut inner_arc = self.inner; - let mut attempts = 0usize; + let unwrap_deadline = Instant::now() + Duration::from_secs(2); let inner = loop { match Arc::try_unwrap(inner_arc) { Ok(inner) => break inner, Err(arc) => { - attempts += 1; - if attempts >= 100 { - panic!( - "Shutdown failed - not all resources are freed \ - after main task is completed" + if Instant::now() >= unwrap_deadline { + tracing::error!( + "Validator shutdown timed out waiting for auxiliary network workers" ); + if let Some(task) = rbc_service_task { + task.abort(); + } + return None; } inner_arc = arc; - tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(1)).await; } } }; // `inner` is now exclusive, so no auxiliary task can enqueue after // this FIFO barrier. Awaiting it keeps the runtime available to the - // RBC actor while any earlier core action completes. - let _ = inner.syncer.missing_parent_references().await; - let mut shadow_shutdown_timed_out = false; - if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { - match tokio::time::timeout(STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT, shadow.shutdown()) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => { - tracing::warn!("RBC-DAG runtime did not acknowledge shutdown: {error}") - } - Err(_) => { - shadow_shutdown_timed_out = true; - tracing::warn!( - "Timed out stopping RBC-DAG runtime; detaching it from validator shutdown" - ); - if let Some(task) = rbc_dag_shadow_service_task.as_ref() { - task.abort(); - } - } + // RBC actor while any earlier core action completes. The barrier is + // fallible because the core thread may concurrently panic while the + // remaining workers still need deterministic cleanup. + let _ = inner.syncer.flush_for_shutdown().await; + let syncer = match inner.syncer.stop() { + Ok(syncer) => Some(syncer), + Err(_) => { + tracing::error!("Core thread terminated before validator shutdown completed"); + None } - } - let syncer = inner.syncer.stop(); + }; if let Some(rbc_service_task) = rbc_service_task { rbc_service_task.abort(); rbc_service_task.await.ok(); } - if let Some(shadow_service_task) = rbc_dag_shadow_service_task { - match shadow_service_task.await { - Err(error) if !shadow_shutdown_timed_out => tracing::warn!( - "Non-authoritative RBC-DAG shadow supervisor failed during shutdown: {error}" - ), - _ => {} - } - } syncer } + pub(crate) fn is_finished(&self) -> bool { + self.main_task.is_finished() || self.inner.syncer.is_finished() + } + async fn run( mut network: Network, universal_committer: UniversalCommitter, @@ -2909,7 +4627,11 @@ impl NetworkSyncer None }; - let commit_timeout_task = handle.spawn(Self::commit_timeout_task(inner.clone())); + // Embedded RBC-DAG frontiers are the sole commit authority. The + // legacy 10 ms commit poll is a no-op in that mode and otherwise + // needlessly submits roughly 100 core-thread commands per second. + let commit_timeout_task = (!inner.embedded_rbc_authority) + .then(|| handle.spawn(Self::commit_timeout_task(inner.clone()))); let cleanup_task = handle.spawn(Self::cleanup_task( inner.clone(), bls_service.clone(), @@ -2962,12 +4684,8 @@ impl NetworkSyncer join_all( connections .into_values() - .chain([ - leader_timeout_task, - commit_timeout_task, - cleanup_task, - missing_parent_pull_task, - ]) + .chain([leader_timeout_task, cleanup_task, missing_parent_pull_task]) + .chain(commit_timeout_task) .chain(soft_block_timeout_task) .chain(cert_pull_task) .chain(round_gap_pull_task), @@ -3020,7 +4738,7 @@ impl NetworkSyncer .peer_senders .write() .insert(peer_id, connection.sender.clone()); - let rbc_outbound_task = inner.starfish_rbc_service.as_ref().map(|_| { + let rbc_outbound_task = inner.starfish_rbc_service.is_some().then(|| { let (rbc_sender, mut rbc_receiver) = mpsc::unbounded_channel(); inner.rbc_peer_senders.write().insert(peer_id, rbc_sender); let network_sender = connection.sender.clone(); @@ -3030,6 +4748,46 @@ impl NetworkSyncer } }) }); + let rbc_dag_outbound_task = inner.starfish_rbc_dag_shadow_service.is_some().then(|| { + let mailbox = RbcDagOutboundMailboxV1::new(); + inner + .rbc_dag_peer_mailboxes + .write() + .insert(peer_id, mailbox.clone()); + let proactive_sender = connection.rbc_dag_proactive_sender.clone(); + let priority_sender = connection.rbc_dag_priority_sender.clone(); + let outbound_failure = connection.outbound_failure.clone(); + let worker_metrics = shadow_metrics.clone(); + let worker_bridge_tx = inner.rbc_dag_clock_bridge_tx.clone(); + let authoritative = inner.embedded_rbc_authority; + Handle::current().spawn(async move { + if let Err(error) = run_rbc_dag_outbound_worker( + mailbox, + proactive_sender, + priority_sender, + outbound_failure, + ) + .await + { + if authoritative { + fail_rbc_dag_authority( + &worker_metrics, + worker_bridge_tx.as_ref(), + &format!( + "RBC-DAG outbound worker for authority {peer_id} failed: {error}" + ), + ); + } else { + invalidate_shadow_run(&worker_metrics); + tracing::error!( + peer = peer_id, + ?error, + "RBC-DAG observational outbound worker failed" + ); + } + } + }) + }); if let Some(ref rbc) = inner.starfish_rbc_service { if let Err(error) = rbc.peer_connected(peer_id) { tracing::warn!( @@ -3112,10 +4870,15 @@ impl NetworkSyncer } inner.peer_senders.write().remove(&peer_id); inner.rbc_peer_senders.write().remove(&peer_id); + inner.rbc_dag_peer_mailboxes.write().remove(&peer_id); if let Some(rbc_outbound_task) = rbc_outbound_task { rbc_outbound_task.abort(); rbc_outbound_task.await.ok(); } + if let Some(rbc_dag_outbound_task) = rbc_dag_outbound_task { + rbc_dag_outbound_task.abort(); + rbc_dag_outbound_task.await.ok(); + } inner.syncer.authority_connection(peer_id, false).await; handler.shutdown().await; block_fetcher.remove_authority(peer_id).await; @@ -3230,6 +4993,9 @@ impl NetworkSyncer inner: Arc>, metrics: Arc, ) -> Option<()> { + if inner.embedded_rbc_authority { + return None; + } const SCAN_INTERVAL: Duration = Duration::from_millis(500); const PEER_COUNT: usize = 2; @@ -3488,15 +5254,397 @@ impl SyncerSignals for NetworkSyncSignals { #[cfg(test)] mod tests { + use std::{collections::VecDeque, sync::Mutex}; + + use prometheus::Registry; use rand::{SeedableRng, rngs::StdRng}; + use tokio::sync::oneshot; use super::*; use crate::{ - crypto::{self, SignatureBytes, TransactionsCommitment}, + crypto::{self, SignatureBytes}, encoder::ShardEncoder, + network::{ + RbcDagShadowCarrier, RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, + }, + starfish_rbc_dag::{ + CarrierHeaderV1Args, ConsensusVertexReference, carrier_genesis_reference, + }, types::{BaseTransaction, BlockReference, Transaction, TransactionData}, }; + fn rbc_dag_outbound_test_carrier( + committee: &Committee, + creation_time_ns: TimestampNs, + ) -> (BlockReference, NetworkMessage) { + let candidate = CandidateCarrierV1::try_new( + CarrierHeaderV1Args { + author: 0, + carrier_round: 1, + own_prev: carrier_genesis_reference(0), + weak_parents: vec![carrier_genesis_reference(1), carrier_genesis_reference(2)], + transactions_commitment: TransactionsCommitment::default(), + application_header: None, + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns, + }, + committee, + ) + .unwrap(); + let reference = candidate.reference(); + ( + reference, + NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { + canonical_carrier: candidate.canonical_wire_bytes().unwrap(), + authentication_sidecar: vec![0xA5], + application_payload: None, + }), + ) + } + + fn rbc_dag_outbound_test_sync_request(round: RoundNumber) -> NetworkMessage { + NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { + author: 1, + round, + }) + } + + fn rbc_dag_outbound_test_sync_response(round: RoundNumber, marker: u8) -> NetworkMessage { + NetworkMessage::RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse { + author: 1, + round, + canonical_carrier: vec![marker], + authentication_sidecar: vec![marker.wrapping_add(1)], + }) + } + + #[derive(Default)] + struct TestRbcDagCoreControlStateV1 { + staged: Vec, + available: Vec<(BlockReference, bool)>, + materialized: Vec, + assignments: Vec, + frontier_results: VecDeque>, + activations: usize, + events: Vec<&'static str>, + } + + #[derive(Default)] + struct TestRbcDagCoreControlTargetV1 { + state: Mutex, + assignment_observed: Notify, + } + + impl RbcDagCoreControlTargetV1 for TestRbcDagCoreControlTargetV1 { + async fn stage_authorized_application( + &self, + header: RbcCanonicalHeader, + _authorization_basis: ShadowApplicationAuthorizationBasisV1, + ) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + state.staged.push(header.reference()); + state.events.push("header"); + Ok(()) + } + + fn restore_available_application( + &self, + application: BlockReference, + payload: Option>, + ) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + state.available.push((application, payload.is_some())); + state.events.push("available"); + Ok(()) + } + + async fn materialize_authorized_payload( + &self, + item: ReconstructedTransactionData, + ) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + state.materialized.push(item.block_reference); + state.events.push("payload"); + Ok(()) + } + + async fn apply_frontier(&self, _delta: CommittedFrontierDeltaV1) -> Result { + let mut state = self.state.lock().unwrap(); + state.events.push("frontier"); + state.frontier_results.pop_front().unwrap_or(Ok(true)) + } + + async fn activate_authority(&self) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + state.activations += 1; + state.events.push("activate"); + Ok(()) + } + + async fn apply_assignment(&self, reference: BlockReference) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + state.assignments.push(reference); + state.events.push("assignment"); + drop(state); + self.assignment_observed.notify_waiters(); + Ok(()) + } + } + + fn rbc_dag_worker_test_metrics() -> Arc { + let registry = Registry::new(); + let (metrics, _) = Metrics::new(®istry, None, Some("starfish-rbc"), None); + metrics.starfish_rbc_dag_shadow_clock_valid.set(-1); + metrics + } + + fn rbc_dag_worker_test_application() -> (RbcCanonicalHeader, ReconstructedTransactionData) { + let committee = Committee::new_test(vec![1; 4]); + let transactions = vec![BaseTransaction::Share(Transaction::new(vec![7; 64]))]; + let mut commitment_encoder = + ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); + let encoded = commitment_encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + let (commitment, _) = TransactionsCommitment::new_from_encoded_transactions(&encoded, 1); + let canonical = RbcCanonicalHeader::try_new( + 0, + 1, + vec![ + BlockReference::new_test(0, 0), + BlockReference::new_test(1, 0), + BlockReference::new_test(2, 0), + ], + Vec::new(), + 11, + commitment, + ) + .unwrap(); + let mut verifier = ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); + let item = verify_starfish_rbc_transaction_payload( + &canonical, + Arc::new(TransactionData::new(transactions)), + &committee, + 1, + 0, + &mut verifier, + BlockAuthenticationScheme::MacVector, + &[], + ) + .unwrap(); + (canonical, item) + } + + fn rbc_dag_worker_test_delta() -> CommittedFrontierDeltaV1 { + let carrier = BlockReference::new_test(0, 1); + CommittedFrontierDeltaV1 { + output_sequence: 1, + anchor: ConsensusVertexReference::new(carrier, 1), + frontier: Vec::new(), + carriers: Vec::new(), + applications: Vec::new(), + application_diagnostics: Vec::new(), + } + } + + #[test] + fn rbc_dag_outbound_mailbox_coalesces_exact_duplicates_and_rejects_conflicts() { + let committee = Committee::new_test(vec![1; 4]); + let mailbox = RbcDagOutboundMailboxV1::new(); + assert_eq!( + mailbox + .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) + .unwrap(), + RbcDagOutboundEnqueueV1::Added + ); + assert_eq!( + mailbox + .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) + .unwrap(), + RbcDagOutboundEnqueueV1::Coalesced + ); + assert!(matches!( + mailbox.enqueue(rbc_dag_outbound_test_sync_response(7, 0xB1), &committee), + Err(RbcDagOutboundMailboxErrorV1::ConflictingDuplicate { + class: RbcDagOutboundClassV1::Priority, + key: RbcDagOutboundKeyV1::SyncResponse(1, 7), + }) + )); + let state = mailbox.inner.state.lock(); + assert_eq!(state.priority.entries.len(), 1); + assert_eq!(state.priority.order.len(), 1); + } + + #[test] + fn rbc_dag_outbound_mailbox_drains_priority_before_proactive() { + let committee = Committee::new_test(vec![1; 4]); + let mailbox = RbcDagOutboundMailboxV1::new(); + let (reference, proactive) = rbc_dag_outbound_test_carrier(&committee, 11); + mailbox.enqueue(proactive, &committee).unwrap(); + mailbox + .enqueue(rbc_dag_outbound_test_sync_request(9), &committee) + .unwrap(); + + let (class, first) = mailbox.try_pop().unwrap(); + assert_eq!(class, RbcDagOutboundClassV1::Priority); + assert!(matches!( + first, + NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { + author: 1, + round: 9, + }) + )); + let (class, second) = mailbox.try_pop().unwrap(); + assert_eq!(class, RbcDagOutboundClassV1::Proactive); + assert!(matches!( + rbc_dag_outbound_classification(&second, &committee).unwrap().1, + RbcDagOutboundKeyV1::Proactive(actual) if actual == reference + )); + } + + #[test] + fn rbc_dag_outbound_mailbox_never_evicts_a_unique_proactive_reference() { + let committee = Committee::new_test(vec![1; 4]); + let mailbox = RbcDagOutboundMailboxV1::new(); + let mut first_reference = None; + for marker in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY { + let (reference, message) = + rbc_dag_outbound_test_carrier(&committee, marker as TimestampNs); + first_reference.get_or_insert(reference); + assert_eq!( + mailbox.enqueue(message, &committee).unwrap(), + RbcDagOutboundEnqueueV1::Added + ); + } + let (_, overflow) = rbc_dag_outbound_test_carrier( + &committee, + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as TimestampNs + 1, + ); + assert!(matches!( + mailbox.enqueue(overflow, &committee), + Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { + class: RbcDagOutboundClassV1::Proactive, + capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + }) + )); + let state = mailbox.inner.state.lock(); + assert_eq!( + state.proactive.entries.len(), + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY + ); + assert!( + state + .proactive + .entries + .contains_key(&RbcDagOutboundKeyV1::Proactive(first_reference.unwrap())) + ); + } + + #[test] + fn rbc_dag_outbound_mailbox_bounds_distinct_priority_keys_and_bytes() { + let committee = Committee::new_test(vec![1; 4]); + let mailbox = RbcDagOutboundMailboxV1::new(); + for round in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber { + mailbox + .enqueue(rbc_dag_outbound_test_sync_request(round), &committee) + .unwrap(); + } + assert!(matches!( + mailbox.enqueue( + rbc_dag_outbound_test_sync_request( + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber + 1, + ), + &committee, + ), + Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { + class: RbcDagOutboundClassV1::Priority, + capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + }) + )); + assert_eq!( + mailbox.inner.state.lock().priority.entries.len(), + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY + ); + + let first = rbc_dag_outbound_test_sync_request(1); + let framed_bytes = usize::try_from(bincode::serialized_size(&first).unwrap()) + .unwrap() + .checked_add(4) + .unwrap(); + let byte_bounded = RbcDagOutboundMailboxV1::from_state(RbcDagOutboundMailboxStateV1 { + priority: RbcDagOutboundLaneV1::new( + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + framed_bytes * 2 - 1, + framed_bytes, + ), + proactive: RbcDagOutboundLaneV1::new( + STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_BYTES, + STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_ENTRY_BYTES, + ), + failure: None, + }); + byte_bounded.enqueue(first, &committee).unwrap(); + assert!(matches!( + byte_bounded.enqueue(rbc_dag_outbound_test_sync_request(2), &committee), + Err(RbcDagOutboundMailboxErrorV1::ByteCapacity { + class: RbcDagOutboundClassV1::Priority, + .. + }) + )); + let state = byte_bounded.inner.state.lock(); + assert_eq!(state.priority.entries.len(), 1); + assert_eq!(state.priority.bytes, framed_bytes); + } + + #[test] + fn rbc_dag_outbound_saturation_fails_authority_or_observation_explicitly() { + let error = RbcDagOutboundMailboxErrorV1::KeyCapacity { + class: RbcDagOutboundClassV1::Proactive, + capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, + }; + let authority_metrics = rbc_dag_worker_test_metrics(); + let (authority_bridge, authority_state) = + watch::channel(RbcDagClockBridgeStateV1::Activated); + fail_rbc_dag_outbound_transport( + &authority_metrics, + Some(&authority_bridge), + true, + 2, + &error, + ); + assert_eq!(*authority_state.borrow(), RbcDagClockBridgeStateV1::Failed); + assert_eq!( + authority_metrics.starfish_rbc_dag_shadow_clock_valid.get(), + 0 + ); + + let observation_metrics = rbc_dag_worker_test_metrics(); + let (observation_bridge, observation_state) = + watch::channel(RbcDagClockBridgeStateV1::Activated); + fail_rbc_dag_outbound_transport( + &observation_metrics, + Some(&observation_bridge), + false, + 2, + &error, + ); + assert_eq!( + *observation_state.borrow(), + RbcDagClockBridgeStateV1::Activated + ); + assert_eq!( + observation_metrics + .starfish_rbc_dag_shadow_clock_valid + .get(), + 0 + ); + } + #[tokio::test] async fn proposal_round_signal_notifies_waiters() { let block_ready_notify = Arc::new(Notify::new()); @@ -3511,6 +5659,475 @@ mod tests { wait.await; } + #[tokio::test] + async fn rbc_dag_clock_bridge_activation_ack_is_fail_closed() { + let (bridge_tx, mut bridge_state) = watch::channel(RbcDagClockBridgeStateV1::Pending); + fail_rbc_dag_clock_bridge(Some(&bridge_tx)); + + let error = wait_for_rbc_dag_clock_bridge_activation(&mut bridge_state) + .await + .unwrap_err(); + assert!(error.contains("startup recovery was invalid")); + } + + #[tokio::test] + async fn rbc_dag_clock_bridge_activation_ack_is_observable_without_a_lost_wakeup() { + let (bridge_tx, mut bridge_state) = watch::channel(RbcDagClockBridgeStateV1::Pending); + bridge_tx.send_replace(RbcDagClockBridgeStateV1::Activated); + + wait_for_rbc_dag_clock_bridge_activation(&mut bridge_state) + .await + .unwrap(); + + // A caller that already observed activation remains returned, while + // every later observer and authority command sees permanent failure. + fail_rbc_dag_clock_bridge(Some(&bridge_tx)); + assert_eq!( + *bridge_state.borrow_and_update(), + RbcDagClockBridgeStateV1::Failed + ); + } + + #[tokio::test] + async fn rbc_dag_assignment_lane_is_not_delayed_by_payload_verification() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, _bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let shutdown_started = Arc::new(AtomicBool::new(false)); + let (control_tx, control_rx) = mpsc::channel(1); + let (assignment_tx, assignment_rx) = mpsc::channel(1); + let control_task = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics.clone(), + bridge_tx.clone(), + shutdown_started.clone(), + control_rx, + )); + let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( + target.clone(), + metrics, + bridge_tx, + shutdown_started, + assignment_rx, + )); + let (release_payload, wait_for_payload) = oneshot::channel::<()>(); + let payload_task = tokio::spawn(async move { + wait_for_payload.await.unwrap(); + Ok(Err(eyre::eyre!("delayed untrusted payload"))) + }); + let (header, _) = rbc_dag_worker_test_application(); + control_tx + .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(1, 2), + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::Verify(payload_task), + }) + .await + .unwrap(); + + let assignment = BlockReference::new_test(0, 3); + let observed = target.assignment_observed.notified(); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Assigned(assignment)) + .await + .unwrap(); + tokio::time::timeout(Duration::from_millis(100), observed) + .await + .expect("assignment must bypass the blocked payload FIFO"); + assert_eq!(target.state.lock().unwrap().assignments, vec![assignment]); + + release_payload.send(()).unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Drain) + .await + .unwrap(); + control_task.await.unwrap(); + assignment_task.await.unwrap(); + } + + #[tokio::test] + async fn rbc_dag_router_fail_fast_blocks_assignment_while_payload_is_slow() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let shutdown_started = Arc::new(AtomicBool::new(false)); + let (control_tx, control_rx) = mpsc::channel(1); + let (assignment_tx, assignment_rx) = mpsc::channel(1); + let control_task = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics.clone(), + bridge_tx.clone(), + shutdown_started.clone(), + control_rx, + )); + let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( + target.clone(), + metrics.clone(), + bridge_tx.clone(), + shutdown_started, + assignment_rx, + )); + let (release_payload, wait_for_payload) = oneshot::channel::<()>(); + let (header, _) = rbc_dag_worker_test_application(); + control_tx + .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(1, 2), + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async move { + wait_for_payload.await.unwrap(); + Ok(Err(eyre::eyre!("slow untrusted payload"))) + })), + }) + .await + .unwrap(); + + // This models a router-known invalid header: failure is published + // synchronously instead of sitting behind the slow FIFO command. + fail_rbc_dag_authority( + &metrics, + Some(&bridge_tx), + "router rejected an invalid authoritative header", + ); + let assignment = BlockReference::new_test(0, 3); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Assigned(assignment)) + .await + .unwrap(); + tokio::task::yield_now().await; + assert_eq!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); + assert!(target.state.lock().unwrap().assignments.is_empty()); + + release_payload.send(()).unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Drain) + .await + .unwrap(); + control_task.await.unwrap(); + assignment_task.await.unwrap(); + } + + #[tokio::test] + async fn rbc_dag_worker_failure_blocks_later_assignment_at_apply_boundary() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + target + .state + .lock() + .unwrap() + .frontier_results + .push_back(Err("rejected frontier".to_owned())); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, mut bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let shutdown_started = Arc::new(AtomicBool::new(false)); + let (control_tx, control_rx) = mpsc::channel(2); + let (assignment_tx, assignment_rx) = mpsc::channel(1); + let control_task = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics.clone(), + bridge_tx.clone(), + shutdown_started.clone(), + control_rx, + )); + let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( + target.clone(), + metrics, + bridge_tx, + shutdown_started, + assignment_rx, + )); + + control_tx + .send(RbcDagCoreControlCommandV1::Frontier( + rbc_dag_worker_test_delta(), + )) + .await + .unwrap(); + tokio::time::timeout(Duration::from_millis(100), async { + while *bridge_rx.borrow_and_update() != RbcDagClockBridgeStateV1::Failed { + bridge_rx.changed().await.unwrap(); + } + }) + .await + .expect("frontier rejection must fail authority promptly"); + + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Assigned( + BlockReference::new_test(0, 3), + )) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Drain) + .await + .unwrap(); + control_task.await.unwrap(); + assignment_task.await.unwrap(); + + assert!(target.state.lock().unwrap().assignments.is_empty()); + } + + #[tokio::test] + async fn rbc_dag_bad_attached_payload_allows_later_valid_materialization() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let (control_tx, control_rx) = mpsc::channel(4); + let worker = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics, + bridge_tx, + Arc::new(AtomicBool::new(false)), + control_rx, + )); + let (header, valid_item) = rbc_dag_worker_test_application(); + let application = header.reference(); + control_tx + .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(1, 2), + header: header.clone(), + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async { + Ok(Err(eyre::eyre!("bad attached bytes"))) + })), + }) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(2, 3), + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async move { + Ok(Ok(valid_item)) + })), + }) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + worker.await.unwrap(); + + assert_ne!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); + let state = target.state.lock().unwrap(); + assert_eq!(state.staged, vec![application, application]); + assert_eq!(state.materialized, vec![application]); + } + + #[tokio::test] + async fn rbc_dag_exact_frontier_replay_has_no_applied_metric_or_effect() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + target + .state + .lock() + .unwrap() + .frontier_results + .push_back(Ok(false)); + let metrics = rbc_dag_worker_test_metrics(); + let applied_before = metrics + .starfish_rbc_dag_frontier_events_total + .with_label_values(&["applied"]) + .get(); + let (bridge_tx, _bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let (control_tx, control_rx) = mpsc::channel(2); + let worker = tokio::spawn(run_rbc_dag_core_control_worker( + target, + metrics.clone(), + bridge_tx, + Arc::new(AtomicBool::new(false)), + control_rx, + )); + control_tx + .send(RbcDagCoreControlCommandV1::Frontier( + rbc_dag_worker_test_delta(), + )) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + worker.await.unwrap(); + assert_eq!( + metrics + .starfish_rbc_dag_frontier_events_total + .with_label_values(&["applied"]) + .get(), + applied_before + ); + } + + #[tokio::test] + async fn rbc_dag_graceful_drain_applies_queued_frontier_before_exit() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let (control_tx, control_rx) = mpsc::channel(4); + let worker = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics, + bridge_tx, + Arc::new(AtomicBool::new(true)), + control_rx, + )); + let (release_payload, wait_for_payload) = oneshot::channel::<()>(); + let (header, item) = rbc_dag_worker_test_application(); + control_tx + .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(1, 2), + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async move { + wait_for_payload.await.unwrap(); + Ok(Ok(item)) + })), + }) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Frontier( + rbc_dag_worker_test_delta(), + )) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + assert!(!worker.is_finished()); + release_payload.send(()).unwrap(); + worker.await.unwrap(); + + assert_ne!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); + assert_eq!( + target.state.lock().unwrap().events, + vec!["header", "payload", "frontier"] + ); + } + + #[tokio::test] + async fn rbc_dag_shutdown_skips_actor_callback_and_assignment_but_applies_frontier() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let shutdown_started = Arc::new(AtomicBool::new(true)); + let (control_tx, control_rx) = mpsc::channel(3); + let (assignment_tx, assignment_rx) = mpsc::channel(1); + let control_task = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics.clone(), + bridge_tx.clone(), + shutdown_started.clone(), + control_rx, + )); + let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( + target.clone(), + metrics, + bridge_tx, + shutdown_started, + assignment_rx, + )); + let (header, _) = rbc_dag_worker_test_application(); + control_tx + .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(1, 2), + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::AlreadyAvailable(None), + }) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Frontier( + rbc_dag_worker_test_delta(), + )) + .await + .unwrap(); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Assigned( + BlockReference::new_test(0, 3), + )) + .await + .unwrap(); + control_tx + .send(RbcDagCoreControlCommandV1::Drain) + .await + .unwrap(); + assignment_tx + .send(RbcDagApplicationAssignmentCommandV1::Drain) + .await + .unwrap(); + control_task.await.unwrap(); + assignment_task.await.unwrap(); + + assert_ne!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); + let state = target.state.lock().unwrap(); + assert!(state.available.is_empty()); + assert!(state.assignments.is_empty()); + assert_eq!(state.events, vec!["header", "frontier"]); + } + + #[tokio::test] + async fn rbc_dag_empty_available_application_precedes_ready_and_activation() { + let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); + let metrics = rbc_dag_worker_test_metrics(); + let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); + let (control_tx, control_rx) = mpsc::channel(4); + let worker = tokio::spawn(run_rbc_dag_core_control_worker( + target.clone(), + metrics, + bridge_tx, + Arc::new(AtomicBool::new(false)), + control_rx, + )); + let header = RbcCanonicalHeader::try_new( + 0, + 1, + vec![ + BlockReference::new_test(0, 0), + BlockReference::new_test(1, 0), + BlockReference::new_test(2, 0), + ], + Vec::new(), + 11, + TransactionsCommitment::default(), + ) + .unwrap(); + let application = header.reference(); + for command in [ + RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { + carrier: BlockReference::new_test(1, 2), + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + payload: RbcDagAuthorizedPayloadV1::AlreadyAvailable(None), + }, + RbcDagCoreControlCommandV1::Ready, + RbcDagCoreControlCommandV1::Activate, + RbcDagCoreControlCommandV1::Drain, + ] { + control_tx.send(command).await.unwrap(); + } + worker.await.unwrap(); + assert_eq!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Activated); + let state = target.state.lock().unwrap(); + assert_eq!(state.available, vec![(application, false)]); + assert_eq!(state.events, vec!["header", "available", "activate"]); + } + #[test] fn starfish_rbc_initial_payload_is_commitment_checked() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 82805482..86856ae5 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -17,7 +17,7 @@ use tokio::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, }, runtime::Handle, - sync::{Mutex, mpsc}, + sync::{mpsc, watch}, time::Instant, }; @@ -32,11 +32,16 @@ use crate::{ stat::HistogramSender, types::{ AuthorityIndex, AuthoritySet, BlockReference, CertMessage, CertMessageKind, PartialSig, - ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, VerifiedBlock, + ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, TransactionData, + VerifiedBlock, }, }; const PING_INTERVAL: Duration = Duration::from_secs(3); +pub(crate) const RBC_DAG_PRIORITY_CHANNEL_CAPACITY: usize = 64; +pub(crate) const RBC_DAG_PROACTIVE_CHANNEL_CAPACITY: usize = 64; +const NETWORK_SCHEDULED_LANE_CAPACITY: usize = 64; +const NETWORK_SCHEDULED_LANE_BYTE_CAPACITY: usize = 256 * 1024 * 1024; // Max buffer size controls the max amount of data (in bytes) to // be sent/received when sending batches of blocks. Based on the @@ -86,13 +91,30 @@ pub struct ShardPayload { /// Non-authoritative Starfish-RBC-DAG carrier used by the persisted shadow /// runtime. Both byte strings use the versioned canonical codecs from /// `starfish_rbc_dag`; the network envelope deliberately adds no second -/// identity or authentication scheme. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +/// identity or authentication scheme. The optional application payload is an +/// untrusted availability sidecar: receivers must verify it against the +/// transaction commitment in the carrier-authenticated application header. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct RbcDagShadowCarrier { pub canonical_carrier: Vec, pub authentication_sidecar: Vec, + pub application_payload: Option>, } +impl PartialEq for RbcDagShadowCarrier { + fn eq(&self, other: &Self) -> bool { + self.canonical_carrier == other.canonical_carrier + && self.authentication_sidecar == other.authentication_sidecar + && match (&self.application_payload, &other.application_payload) { + (Some(left), Some(right)) => left.transactions() == right.transactions(), + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } + } +} + +impl Eq for RbcDagShadowCarrier {} + /// Content-only response for a phase-evidenced shadow carrier. Recovery can /// satisfy READY/delivery, but it cannot grant optimistic admission or ECHO. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] @@ -121,6 +143,15 @@ pub struct RbcDagShadowCarrierSyncResponse { pub authentication_sidecar: Vec, } +/// Full transaction payload for one application header already authorized by +/// an authenticated or phase-evidenced embedded carrier. The payload itself +/// grants no authority and must be checked against the header commitment. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RbcDagApplicationPayloadResponse { + pub application: BlockReference, + pub transaction_data: Arc, +} + /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -242,6 +273,12 @@ pub enum NetworkMessage { /// Full canonical carrier and authentication sidecar for an exact /// carrier-clock slot. Receivers validate the duplicated slot identity. RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse), + /// Request transaction data for one exact application header already + /// authorized through the embedded carrier protocol. + RbcDagApplicationPayloadRequest(BlockReference), + /// Return commitment-checked transaction data for an authorized embedded + /// application header. The response is not an author proof. + RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse), } impl NetworkMessage { @@ -274,6 +311,8 @@ impl NetworkMessage { Self::RbcDagShadowCarrierResponse(_) => "rbc_dag_shadow_carrier_response", Self::RbcDagShadowCarrierSyncRequest(_) => "rbc_dag_shadow_carrier_sync_request", Self::RbcDagShadowCarrierSyncResponse(_) => "rbc_dag_shadow_carrier_sync_response", + Self::RbcDagApplicationPayloadRequest(_) => "rbc_dag_application_payload_request", + Self::RbcDagApplicationPayloadResponse(_) => "rbc_dag_application_payload_response", } } } @@ -286,9 +325,31 @@ pub struct Network { pub struct Connection { pub peer_id: usize, pub sender: mpsc::Sender, + /// Exact RBC-DAG repair remains distinct from ordinary/proactive traffic + /// until the socket scheduler makes its final priority decision. + pub(crate) rbc_dag_priority_sender: mpsc::Sender, + /// Proactive RBC-DAG carriers remain bounded independently of the + /// connection's legacy ordinary channel. + pub(crate) rbc_dag_proactive_sender: mpsc::Sender, + /// Terminal writer/scheduler failure for fail-closed authoritative users. + pub(crate) outbound_failure: watch::Receiver>, + /// Keep the connection-scoped failure signal open while this public + /// connection is alive. Cancelling the writer because the read half + /// disconnected or a newer socket replaced it is not a terminal writer + /// failure and must not look like one to authoritative users. + _outbound_failure_lifetime: watch::Sender>, pub receiver: mpsc::Receiver, } +impl Drop for Network { + fn drop(&mut self) { + // Dropping a Tokio JoinHandle detaches its task. Abort explicitly so + // a panic in the network-sync main loop cannot leave the listener + // alive and sharing SO_REUSEPORT with a later benchmark. + self.server_task.abort(); + } +} + impl Network { pub async fn load( parameters: &NodePublicConfig, @@ -317,6 +378,12 @@ impl Network { self.server_task.abort(); } + #[cfg(test)] + async fn abort_and_wait(mut self) -> Result<(), tokio::task::JoinError> { + self.server_task.abort(); + (&mut self.server_task).await + } + pub async fn from_socket_addresses( addresses: &[SocketAddr], our_id: usize, @@ -430,6 +497,196 @@ fn bind_addr(mut local_peer: SocketAddr) -> SocketAddr { const NETWORK_MESSAGE_CHANNEL_CAPACITY: usize = 1_000; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ScheduledNetworkClass { + Priority, + Ordinary, +} + +enum ScheduledNetworkPayload { + Message { + wire_bytes: Vec, + request_type: &'static str, + }, + Ping([u8; 12]), +} + +impl ScheduledNetworkPayload { + fn len(&self) -> usize { + match self { + Self::Message { wire_bytes, .. } => wire_bytes.len().saturating_add(4), + Self::Ping(bytes) => bytes.len(), + } + } +} + +#[cfg(test)] +fn scheduled_ping_value(payload: &ScheduledNetworkPayload) -> Option { + match payload { + ScheduledNetworkPayload::Ping(bytes) => Some(decode_ping(&bytes[4..])), + ScheduledNetworkPayload::Message { .. } => None, + } +} + +struct ScheduledNetworkWrite { + ready_at: Instant, + sequence: u64, + payload: ScheduledNetworkPayload, +} + +struct ScheduledNetworkLane { + writes: Vec, + bytes: usize, + count_capacity: usize, + byte_capacity: usize, +} + +impl ScheduledNetworkLane { + fn new(count_capacity: usize, byte_capacity: usize) -> Self { + Self { + writes: Vec::new(), + bytes: 0, + count_capacity, + byte_capacity, + } + } + + /// Reserve enough room for the largest legal frame before receiving the + /// next opaque `NetworkMessage`. This makes the post-serialization byte + /// bound strict without pulling an item out of a backpressured channel + /// that cannot yet be admitted. + fn can_receive_max_frame(&self) -> bool { + const MAX_FRAMED_BYTES: usize = MAX_BUFFER_SIZE as usize + 4; + self.writes.len() < self.count_capacity + && self.bytes <= self.byte_capacity.saturating_sub(MAX_FRAMED_BYTES) + } + + fn push(&mut self, write: ScheduledNetworkWrite) -> Result<(), ScheduledNetworkWrite> { + let bytes = write.payload.len(); + let Some(next_bytes) = self.bytes.checked_add(bytes) else { + return Err(write); + }; + if self.writes.len() >= self.count_capacity || next_bytes > self.byte_capacity { + return Err(write); + } + self.bytes = next_bytes; + self.writes.push(write); + Ok(()) + } + + fn ready_index(&self, now: Instant) -> Option { + self.writes + .iter() + .enumerate() + .filter(|(_, write)| write.ready_at <= now) + .min_by_key(|(_, write)| (write.ready_at, write.sequence)) + .map(|(index, _)| index) + } + + fn pop_ready(&mut self, now: Instant) -> Option { + let index = self.ready_index(now)?; + let write = self.writes.swap_remove(index); + self.bytes = self + .bytes + .checked_sub(write.payload.len()) + .expect("scheduled network byte accounting cannot underflow"); + Some(write) + } + + fn next_deadline(&self) -> Option { + self.writes.iter().map(|write| write.ready_at).min() + } + + fn is_empty(&self) -> bool { + self.writes.is_empty() + } +} + +struct ScheduledNetworkWrites { + priority: ScheduledNetworkLane, + ordinary: ScheduledNetworkLane, + next_sequence: u64, +} + +impl ScheduledNetworkWrites { + fn production() -> Self { + Self::new( + NETWORK_SCHEDULED_LANE_CAPACITY, + NETWORK_SCHEDULED_LANE_BYTE_CAPACITY, + ) + } + + fn new(lane_count_capacity: usize, lane_byte_capacity: usize) -> Self { + Self { + priority: ScheduledNetworkLane::new(lane_count_capacity, lane_byte_capacity), + ordinary: ScheduledNetworkLane::new(lane_count_capacity, lane_byte_capacity), + next_sequence: 0, + } + } + + fn lane(&self, class: ScheduledNetworkClass) -> &ScheduledNetworkLane { + match class { + ScheduledNetworkClass::Priority => &self.priority, + ScheduledNetworkClass::Ordinary => &self.ordinary, + } + } + + fn lane_mut(&mut self, class: ScheduledNetworkClass) -> &mut ScheduledNetworkLane { + match class { + ScheduledNetworkClass::Priority => &mut self.priority, + ScheduledNetworkClass::Ordinary => &mut self.ordinary, + } + } + + fn can_receive(&self, class: ScheduledNetworkClass) -> bool { + self.lane(class).can_receive_max_frame() + } + + fn push( + &mut self, + class: ScheduledNetworkClass, + ready_at: Instant, + payload: ScheduledNetworkPayload, + ) -> io::Result<()> { + let sequence = self.next_sequence; + self.next_sequence = self.next_sequence.saturating_add(1); + self.lane_mut(class) + .push(ScheduledNetworkWrite { + ready_at, + sequence, + payload, + }) + .map_err(|write| { + io::Error::new( + io::ErrorKind::OutOfMemory, + format!( + "scheduled {:?} network lane exhausted at {} bytes", + class, + write.payload.len() + ), + ) + }) + } + + fn pop_ready(&mut self, now: Instant) -> Option { + self.priority + .pop_ready(now) + .or_else(|| self.ordinary.pop_ready(now)) + } + + fn next_deadline(&self) -> Option { + match (self.priority.next_deadline(), self.ordinary.next_deadline()) { + (Some(left), Some(right)) => Some(left.min(right)), + (Some(deadline), None) | (None, Some(deadline)) => Some(deadline), + (None, None) => None, + } + } + + fn is_empty(&self) -> bool { + self.priority.is_empty() && self.ordinary.is_empty() + } +} + struct Worker { peer: SocketAddr, peer_id: usize, @@ -445,11 +702,103 @@ struct Worker { struct WorkerConnection { sender: mpsc::Sender, receiver: mpsc::Receiver, + rbc_dag_priority_receiver: mpsc::Receiver, + rbc_dag_proactive_receiver: mpsc::Receiver, + outbound_failure: watch::Sender>, metrics: Arc, peer_id: usize, compress_network: bool, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OrdinaryAdmissionSource { + Proactive, + Legacy, +} + +impl OrdinaryAdmissionSource { + fn alternate(self) -> Self { + match self { + Self::Proactive => Self::Legacy, + Self::Legacy => Self::Proactive, + } + } + + fn try_receive( + self, + proactive_receiver: &mut mpsc::Receiver, + legacy_receiver: &mut mpsc::Receiver, + ) -> Result { + match self { + Self::Proactive => proactive_receiver.try_recv(), + Self::Legacy => legacy_receiver.try_recv(), + } + } +} + +fn admit_ordinary_fairly( + scheduled: &mut ScheduledNetworkWrites, + proactive_receiver: &mut mpsc::Receiver, + legacy_receiver: &mut mpsc::Receiver, + proactive_closed: &mut bool, + legacy_closed: &mut bool, + next_source: &mut OrdinaryAdmissionSource, + effective_latency: f64, + compress_network: bool, + metrics: &Metrics, +) -> io::Result { + if !scheduled.can_receive(ScheduledNetworkClass::Ordinary) { + return Ok(false); + } + let Some(message) = try_receive_ordinary_fairly( + proactive_receiver, + legacy_receiver, + proactive_closed, + legacy_closed, + next_source, + ) else { + return Ok(false); + }; + schedule_network_message( + scheduled, + ScheduledNetworkClass::Ordinary, + message, + effective_latency, + compress_network, + metrics, + )?; + Ok(true) +} + +fn try_receive_ordinary_fairly( + proactive_receiver: &mut mpsc::Receiver, + legacy_receiver: &mut mpsc::Receiver, + proactive_closed: &mut bool, + legacy_closed: &mut bool, + next_source: &mut OrdinaryAdmissionSource, +) -> Option { + for source in [*next_source, next_source.alternate()] { + if match source { + OrdinaryAdmissionSource::Proactive => *proactive_closed, + OrdinaryAdmissionSource::Legacy => *legacy_closed, + } { + continue; + } + match source.try_receive(proactive_receiver, legacy_receiver) { + Ok(message) => { + *next_source = source.alternate(); + return Some(message); + } + Err(mpsc::error::TryRecvError::Empty) => {} + Err(mpsc::error::TryRecvError::Disconnected) => match source { + OrdinaryAdmissionSource::Proactive => *proactive_closed = true, + OrdinaryAdmissionSource::Legacy => *legacy_closed = true, + }, + } + } + None +} + impl Worker { const ACTIVE_HANDSHAKE: u64 = 0xFEFE0000; const PASSIVE_HANDSHAKE: u64 = 0x0000AEAE; @@ -546,6 +895,9 @@ impl Worker { let WorkerConnection { sender, receiver, + rbc_dag_priority_receiver, + rbc_dag_proactive_receiver, + outbound_failure, metrics, peer_id, compress_network, @@ -562,16 +914,26 @@ impl Worker { correct committee?", ) .clone(); - let write_fut = Self::handle_write_stream( - writer, - receiver, - pong_receiver, - latency_sender, - metrics.clone(), - extra_connection_latency, - extra_connection_scaled, - compress_network, - ) + let write_metrics = metrics.clone(); + let write_fut = async move { + let result = Self::handle_write_stream( + writer, + receiver, + rbc_dag_priority_receiver, + rbc_dag_proactive_receiver, + pong_receiver, + latency_sender, + write_metrics, + extra_connection_latency, + extra_connection_scaled, + compress_network, + ) + .await; + if let Err(error) = &result { + outbound_failure.send_replace(Some(error.to_string())); + } + result + } .boxed(); let read_fut = Self::handle_read_stream(reader, sender, pong_sender, metrics, compress_network) @@ -582,8 +944,10 @@ impl Worker { } async fn handle_write_stream( - writer: OwnedWriteHalf, + mut writer: OwnedWriteHalf, mut receiver: mpsc::Receiver, + mut rbc_dag_priority_receiver: mpsc::Receiver, + mut rbc_dag_proactive_receiver: mpsc::Receiver, mut pong_receiver: mpsc::Receiver, latency_sender: HistogramSender, metrics: Arc, @@ -591,209 +955,147 @@ impl Worker { connection_scaled: bool, compress_network: bool, ) -> io::Result<()> { - // Use Arc and Mutex to share the writer safely across multiple tasks - let writer = Arc::new(Mutex::new(writer)); let start = Instant::now(); - let bytes_sent_total = metrics.bytes_sent_total.clone(); - let network_requests_sent_total = metrics.network_requests_sent_total.clone(); - let network_message_bytes_sent_total = metrics.network_message_bytes_sent_total.clone(); - - // Spawn the first task for handling pings - let writer_clone = Arc::clone(&writer); - let bytes_sent_total_clone = bytes_sent_total.clone(); - let ping_task = async move { - let mut ping_deadline = start + PING_INTERVAL; - loop { - tokio::time::sleep_until(ping_deadline).await; - ping_deadline += PING_INTERVAL; - - let ping_time = start.elapsed().as_micros() as i64; - assert!(ping_time > 0); // interval can't be 0 - - let ping = encode_ping(ping_time); - let latency = - generate_latency(effective_latency(connection_latency, connection_scaled)); - tokio::time::sleep(latency).await; - - if let Err(e) = writer_clone.lock().await.write_all(&ping).await { - tracing::error!("Failed to write ping: {e}"); - break; - } - bytes_sent_total_clone.inc_by(12); // ping is 12-byte sized - } - }; + let effective_latency = effective_latency(connection_latency, connection_scaled); + let mut scheduled = ScheduledNetworkWrites::production(); + let mut ordinary_closed = false; + let mut priority_closed = false; + let mut proactive_closed = false; + let mut pong_closed = false; + let mut ping_deadline = start + PING_INTERVAL; + let mut next_ordinary_source = OrdinaryAdmissionSource::Proactive; - // Spawn the second task for handling pong responses - let writer_clone = Arc::clone(&writer); - let bytes_sent_total_clone = bytes_sent_total.clone(); - let pong_task = async move { - while let Some(ping) = pong_receiver.recv().await { - if ping == 0 { - tracing::warn!("Invalid ping: {ping}"); - break; - } - if ping > 0 { - match ping.checked_neg() { - Some(pong) => { - let pong = encode_ping(pong); - let latency = generate_latency(effective_latency( - connection_latency, - connection_scaled, - )); - tokio::time::sleep(latency).await; - - if let Err(e) = writer_clone.lock().await.write_all(&pong).await { - tracing::error!("Failed to write pong: {e}"); - break; - } - bytes_sent_total_clone.inc_by(12); // pong is 12-byte sized - } - None => { - tracing::warn!("Invalid ping: {ping}"); - break; - } - } - } else { - match ping.checked_neg().and_then(|n| u64::try_from(n).ok()) { - Some(our_ping) => { - let time = start.elapsed().as_micros() as u64; - if let Some(delay) = time.checked_sub(our_ping) { - latency_sender.observe(Duration::from_micros(delay)); - } else { - tracing::warn!("Invalid ping: {ping}, greater than current time"); - break; - } - } - None => { - tracing::warn!("Invalid pong: {ping}"); - break; - } + loop { + // Keep socket liveness independent of application throughput. A + // continuously ready zero-latency data lane must not prevent the + // read half from handing us pings (and eventually blocking on its + // bounded pong channel), nor postpone our periodic ping forever. + let now = Instant::now(); + service_network_keepalive( + &mut scheduled, + &mut pong_receiver, + &mut pong_closed, + start, + now, + &mut ping_deadline, + effective_latency, + &latency_sender, + )?; + + // Always admit queued exact repair before ordinary traffic. The + // two scheduled lanes have independent 64-item/byte bounds, so a + // proactive burst cannot consume repair capacity. + while scheduled.can_receive(ScheduledNetworkClass::Priority) { + match rbc_dag_priority_receiver.try_recv() { + Ok(message) => schedule_network_message( + &mut scheduled, + ScheduledNetworkClass::Priority, + message, + effective_latency, + compress_network, + &metrics, + )?, + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => { + priority_closed = true; + break; } } - // Yield to ensure responsiveness - tokio::task::yield_now().await; } - }; + while admit_ordinary_fairly( + &mut scheduled, + &mut rbc_dag_proactive_receiver, + &mut receiver, + &mut proactive_closed, + &mut ordinary_closed, + &mut next_ordinary_source, + effective_latency, + compress_network, + &metrics, + )? {} + + if let Some(write) = scheduled.pop_ready(Instant::now()) { + write_scheduled_network_payload(&mut writer, write.payload, &metrics).await?; + continue; + } + if ordinary_closed && priority_closed && proactive_closed && scheduled.is_empty() { + return Ok(()); + } - // Spawn the third task(s) for handling message sending. - // - // Important: keep encoding (serialize/compress) decoupled from timed - // socket writes. If we combine them, bursts of large writes can - // backpressure encoding and starve catch-up for late joiners under - // latency simulation. - if connection_latency == 0.0 { - let message_task = async move { - while let Some(message) = receiver.recv().await { - let request_type = message.request_type(); - let serialized = bincode::serialize(&message).expect("Serialization failed"); - let wire_bytes = if compress_network { - metrics - .bytes_uncompressed_sent_total - .inc_by(serialized.len() as u64); - lz4_flex::compress_prepend_size(&serialized) - } else { - serialized - }; - let framed_len = wire_bytes.len() as u64 + 4; - - match async { - let mut writer_guard = writer.lock().await; - writer_guard.write_u32(wire_bytes.len() as u32).await?; - - bytes_sent_total.inc_by(framed_len); - writer_guard.write_all(&wire_bytes).await - } - .await - { - Ok(()) => { - network_requests_sent_total - .with_label_values(&[request_type]) - .inc(); - network_message_bytes_sent_total - .with_label_values(&[request_type]) - .inc_by(framed_len); - } - Err(e) => { - tracing::error!("Failed to write message: {e}"); - } + let next_write_deadline = scheduled.next_deadline(); + let next_deadline = + next_write_deadline.map_or(ping_deadline, |deadline| deadline.min(ping_deadline)); + tokio::select! { + biased; + message = rbc_dag_priority_receiver.recv(), + if !priority_closed + && scheduled.can_receive(ScheduledNetworkClass::Priority) => { + match message { + Some(message) => schedule_network_message( + &mut scheduled, + ScheduledNetworkClass::Priority, + message, + effective_latency, + compress_network, + &metrics, + )?, + None => priority_closed = true, } } - }; - - // Wait for all tasks to complete. - let _ = tokio::join!(ping_task, pong_task, message_task); - return Ok(()); - } - - // When simulating latency, preserve the "many in-flight messages" - // behavior by sleeping inside per-message tasks, but keep concurrency - // bounded to avoid unbounded task buildup under heavy load. - const MAX_IN_FLIGHT: usize = NETWORK_MESSAGE_CHANNEL_CAPACITY * 64; - let message_task = async move { - let mut join_set = tokio::task::JoinSet::new(); - - while let Some(message) = receiver.recv().await { - while join_set.len() >= MAX_IN_FLIGHT { - if join_set.join_next().await.is_none() { - break; + pong = pong_receiver.recv(), + if !pong_closed + && scheduled.can_receive(ScheduledNetworkClass::Priority) => { + match pong { + Some(pong) => schedule_or_observe_pong( + &mut scheduled, + pong, + start, + Instant::now(), + effective_latency, + &latency_sender, + )?, + None => pong_closed = true, } } - - let writer = writer.clone(); - let bytes_sent_total = bytes_sent_total.clone(); - let network_requests_sent_total = network_requests_sent_total.clone(); - let network_message_bytes_sent_total = network_message_bytes_sent_total.clone(); - let bytes_uncompressed_sent_total = metrics.bytes_uncompressed_sent_total.clone(); - let latency = - generate_latency(effective_latency(connection_latency, connection_scaled)); - let request_type = message.request_type(); - - join_set.spawn(async move { - let serialized = bincode::serialize(&message).expect("Serialization failed"); - let wire_bytes = if compress_network { - bytes_uncompressed_sent_total.inc_by(serialized.len() as u64); - lz4_flex::compress_prepend_size(&serialized) - } else { - serialized - }; - let framed_len = wire_bytes.len() as u64 + 4; - tokio::time::sleep(latency).await; - - match async { - let mut writer_guard = writer.lock().await; - writer_guard.write_u32(wire_bytes.len() as u32).await?; - - bytes_sent_total.inc_by(framed_len); - writer_guard.write_all(&wire_bytes).await - } - .await - { - Ok(()) => { - network_requests_sent_total - .with_label_values(&[request_type]) - .inc(); - network_message_bytes_sent_total - .with_label_values(&[request_type]) - .inc_by(framed_len); + message = rbc_dag_proactive_receiver.recv(), + if !proactive_closed + && scheduled.can_receive(ScheduledNetworkClass::Ordinary) => { + match message { + Some(message) => { + schedule_network_message( + &mut scheduled, + ScheduledNetworkClass::Ordinary, + message, + effective_latency, + compress_network, + &metrics, + )?; + next_ordinary_source = OrdinaryAdmissionSource::Legacy; } - Err(e) => { - tracing::error!("Failed to write message: {e}"); + None => proactive_closed = true, + } + } + message = receiver.recv(), + if !ordinary_closed + && scheduled.can_receive(ScheduledNetworkClass::Ordinary) => { + match message { + Some(message) => { + schedule_network_message( + &mut scheduled, + ScheduledNetworkClass::Ordinary, + message, + effective_latency, + compress_network, + &metrics, + )?; + next_ordinary_source = OrdinaryAdmissionSource::Proactive; } + None => ordinary_closed = true, } - }); - } - - while let Some(result) = join_set.join_next().await { - if let Err(e) = result { - tracing::error!("An inner task failed: {e:?}"); } + _ = tokio::time::sleep_until(next_deadline) => {} } - }; - - // Wait for all tasks to complete. - let _ = tokio::join!(ping_task, pong_task, message_task); - - Ok(()) + } } async fn handle_read_stream( @@ -878,15 +1180,27 @@ impl Worker { mpsc::channel(NETWORK_MESSAGE_CHANNEL_CAPACITY); let (network_out_sender, network_out_receiver) = mpsc::channel(NETWORK_MESSAGE_CHANNEL_CAPACITY); + let (rbc_dag_priority_sender, rbc_dag_priority_receiver) = + mpsc::channel(RBC_DAG_PRIORITY_CHANNEL_CAPACITY); + let (rbc_dag_proactive_sender, rbc_dag_proactive_receiver) = + mpsc::channel(RBC_DAG_PROACTIVE_CHANNEL_CAPACITY); + let (outbound_failure, outbound_failure_receiver) = watch::channel(None); let connection = Connection { peer_id: self.peer_id, sender: network_out_sender, + rbc_dag_priority_sender, + rbc_dag_proactive_sender, + outbound_failure: outbound_failure_receiver, + _outbound_failure_lifetime: outbound_failure.clone(), receiver: network_in_receiver, }; self.connection_sender.send(connection).await.ok()?; Some(WorkerConnection { sender: network_in_sender, receiver: network_out_receiver, + rbc_dag_priority_receiver, + rbc_dag_proactive_receiver, + outbound_failure, metrics: self.metrics.clone(), peer_id: self.peer_id, compress_network: self.compress_network, @@ -894,6 +1208,165 @@ impl Worker { } } +fn schedule_network_message( + scheduled: &mut ScheduledNetworkWrites, + class: ScheduledNetworkClass, + message: NetworkMessage, + effective_latency: f64, + compress_network: bool, + metrics: &Metrics, +) -> io::Result<()> { + let request_type = message.request_type(); + let serialized = bincode::serialize(&message).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("network message serialization failed: {error}"), + ) + })?; + let wire_bytes = if compress_network { + metrics + .bytes_uncompressed_sent_total + .inc_by(serialized.len() as u64); + lz4_flex::compress_prepend_size(&serialized) + } else { + serialized + }; + if wire_bytes.len() > MAX_BUFFER_SIZE as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "serialized network message is {} bytes, maximum {}", + wire_bytes.len(), + MAX_BUFFER_SIZE + ), + )); + } + scheduled.push( + class, + Instant::now() + generate_latency(effective_latency), + ScheduledNetworkPayload::Message { + wire_bytes, + request_type, + }, + ) +} + +fn schedule_or_observe_pong( + scheduled: &mut ScheduledNetworkWrites, + ping: i64, + start: Instant, + now: Instant, + effective_latency: f64, + latency_sender: &HistogramSender, +) -> io::Result<()> { + if ping == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "received a zero ping", + )); + } + if ping > 0 { + let pong = ping + .checked_neg() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "ping cannot be negated"))?; + return scheduled.push( + ScheduledNetworkClass::Priority, + now + generate_latency(effective_latency), + ScheduledNetworkPayload::Ping(encode_ping(pong)), + ); + } + let our_ping = ping + .checked_neg() + .and_then(|value| u64::try_from(value).ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid pong"))?; + let elapsed = now.saturating_duration_since(start).as_micros() as u64; + let delay = elapsed.checked_sub(our_ping).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "pong timestamp is greater than the local clock", + ) + })?; + latency_sender.observe(Duration::from_micros(delay)); + Ok(()) +} + +fn service_network_keepalive( + scheduled: &mut ScheduledNetworkWrites, + pong_receiver: &mut mpsc::Receiver, + pong_closed: &mut bool, + start: Instant, + now: Instant, + ping_deadline: &mut Instant, + effective_latency: f64, + latency_sender: &HistogramSender, +) -> io::Result<()> { + if !*pong_closed && scheduled.can_receive(ScheduledNetworkClass::Priority) { + match pong_receiver.try_recv() { + Ok(pong) => schedule_or_observe_pong( + scheduled, + pong, + start, + now, + effective_latency, + latency_sender, + )?, + Err(mpsc::error::TryRecvError::Empty) => {} + Err(mpsc::error::TryRecvError::Disconnected) => *pong_closed = true, + } + } + if now >= *ping_deadline { + if scheduled.can_receive(ScheduledNetworkClass::Priority) { + let ping_time = now.saturating_duration_since(start).as_micros() as i64; + if ping_time > 0 { + scheduled.push( + ScheduledNetworkClass::Priority, + now + generate_latency(effective_latency), + ScheduledNetworkPayload::Ping(encode_ping(ping_time)), + )?; + } + } + *ping_deadline = now + PING_INTERVAL; + } + Ok(()) +} + +async fn write_scheduled_network_payload( + writer: &mut OwnedWriteHalf, + payload: ScheduledNetworkPayload, + metrics: &Metrics, +) -> io::Result<()> { + match payload { + ScheduledNetworkPayload::Message { + wire_bytes, + request_type, + } => { + let wire_len = u32::try_from(wire_bytes.len()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "network frame length exceeds u32", + ) + })?; + writer.write_u32(wire_len).await?; + writer.write_all(&wire_bytes).await?; + let framed_len = wire_bytes.len() as u64 + 4; + metrics.bytes_sent_total.inc_by(framed_len); + metrics + .network_requests_sent_total + .with_label_values(&[request_type]) + .inc(); + metrics + .network_message_bytes_sent_total + .with_label_values(&[request_type]) + .inc_by(framed_len); + } + ScheduledNetworkPayload::Ping(bytes) => { + writer.write_all(&bytes).await?; + metrics.bytes_sent_total.inc_by(bytes.len() as u64); + } + } + Ok(()) +} + /// Generates a latency table for a geodistributed network. /// `n` is the number of nodes. /// `seed` is a global seed used for deterministic generation. @@ -1024,12 +1497,310 @@ mod tests { use super::*; use crate::{ committee::Committee, - crypto::{MacTag, TransactionsCommitment, dummy_signer}, + crypto::{AsBytes, MacTag, TransactionsCommitment, dummy_signer}, starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, + types::{BaseTransaction, Transaction}, }; const NETWORK_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(6); + fn scheduled_test_message(marker: RoundNumber) -> ScheduledNetworkPayload { + let message = NetworkMessage::SubscribeBroadcastRequest(marker); + ScheduledNetworkPayload::Message { + wire_bytes: bincode::serialize(&message).unwrap(), + request_type: message.request_type(), + } + } + + fn scheduled_test_marker(write: ScheduledNetworkWrite) -> RoundNumber { + let ScheduledNetworkPayload::Message { wire_bytes, .. } = write.payload else { + panic!("expected a scheduled network message"); + }; + let NetworkMessage::SubscribeBroadcastRequest(marker) = + bincode::deserialize(&wire_bytes).unwrap() + else { + panic!("expected a scheduled subscription marker"); + }; + marker + } + + fn outbound_test_marker(message: NetworkMessage) -> RoundNumber { + let NetworkMessage::SubscribeBroadcastRequest(marker) = message else { + panic!("expected an outbound subscription marker"); + }; + marker + } + + #[test] + fn ordinary_admission_alternates_continuously_ready_sources() { + let (proactive_sender, mut proactive_receiver) = mpsc::channel(8); + let (legacy_sender, mut legacy_receiver) = mpsc::channel(8); + for marker in 10..14 { + proactive_sender + .try_send(NetworkMessage::SubscribeBroadcastRequest(marker)) + .unwrap(); + } + for marker in 20..24 { + legacy_sender + .try_send(NetworkMessage::SubscribeBroadcastRequest(marker)) + .unwrap(); + } + let mut proactive_closed = false; + let mut legacy_closed = false; + let mut next = OrdinaryAdmissionSource::Proactive; + let markers = (0..8) + .map(|_| { + outbound_test_marker( + try_receive_ordinary_fairly( + &mut proactive_receiver, + &mut legacy_receiver, + &mut proactive_closed, + &mut legacy_closed, + &mut next, + ) + .expect("both saturated sources should remain admissible"), + ) + }) + .collect::>(); + assert_eq!(markers, vec![10, 20, 11, 21, 12, 22, 13, 23]); + } + + #[test] + fn ordinary_admission_falls_through_without_losing_its_fair_turn() { + let (proactive_sender, mut proactive_receiver) = mpsc::channel(4); + let (legacy_sender, mut legacy_receiver) = mpsc::channel(4); + legacy_sender + .try_send(NetworkMessage::SubscribeBroadcastRequest(20)) + .unwrap(); + let mut proactive_closed = false; + let mut legacy_closed = false; + let mut next = OrdinaryAdmissionSource::Proactive; + assert_eq!( + outbound_test_marker( + try_receive_ordinary_fairly( + &mut proactive_receiver, + &mut legacy_receiver, + &mut proactive_closed, + &mut legacy_closed, + &mut next, + ) + .unwrap(), + ), + 20 + ); + proactive_sender + .try_send(NetworkMessage::SubscribeBroadcastRequest(10)) + .unwrap(); + assert_eq!( + outbound_test_marker( + try_receive_ordinary_fairly( + &mut proactive_receiver, + &mut legacy_receiver, + &mut proactive_closed, + &mut legacy_closed, + &mut next, + ) + .unwrap(), + ), + 10 + ); + } + + #[test] + fn keepalive_is_serviced_before_a_continuously_ready_data_lane() { + let (mut histogram, latency_sender) = crate::stat::histogram::(); + let (pong_sender, mut pong_receiver) = mpsc::channel(16); + pong_sender.try_send(7).unwrap(); + let start = Instant::now(); + let now = start + PING_INTERVAL; + let mut ping_deadline = now; + let mut pong_closed = false; + let mut scheduled = ScheduledNetworkWrites::new(64, NETWORK_SCHEDULED_LANE_BYTE_CAPACITY); + scheduled + .push( + ScheduledNetworkClass::Ordinary, + start, + scheduled_test_message(10), + ) + .unwrap(); + + service_network_keepalive( + &mut scheduled, + &mut pong_receiver, + &mut pong_closed, + start, + now, + &mut ping_deadline, + 0.0, + &latency_sender, + ) + .unwrap(); + + let first = scheduled.pop_ready(now).unwrap(); + assert_eq!(scheduled_ping_value(&first.payload), Some(-7)); + let second = scheduled.pop_ready(now).unwrap(); + assert_eq!( + scheduled_ping_value(&second.payload), + Some(PING_INTERVAL.as_micros() as i64) + ); + assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 10); + histogram.receive_all(); + assert_eq!(histogram.total_count(), 0); + assert_eq!(ping_deadline, now + PING_INTERVAL); + } + + #[test] + fn normal_writer_cancellation_does_not_close_failure_signal() { + let (writer_failure, worker_failure) = watch::channel(None::); + let connection_lifetime = writer_failure.clone(); + drop(writer_failure); + assert!(matches!(worker_failure.has_changed(), Ok(false))); + drop(connection_lifetime); + assert!(worker_failure.has_changed().is_err()); + } + + #[test] + fn scheduled_writer_preserves_ready_priority_under_zero_and_aws_latency() { + let now = Instant::now(); + for latency in [Duration::ZERO, Duration::from_millis(130)] { + let mut scheduled = ScheduledNetworkWrites::new(64, 1024 * 1024); + scheduled + .push( + ScheduledNetworkClass::Ordinary, + now + latency, + scheduled_test_message(10), + ) + .unwrap(); + scheduled + .push( + ScheduledNetworkClass::Priority, + now + latency, + scheduled_test_message(20), + ) + .unwrap(); + assert_eq!( + scheduled_test_marker(scheduled.pop_ready(now + latency).unwrap()), + 20 + ); + assert_eq!( + scheduled_test_marker(scheduled.pop_ready(now + latency).unwrap()), + 10 + ); + } + } + + #[test] + fn scheduled_writer_does_not_send_unready_priority_ahead_of_ready_ordinary() { + let now = Instant::now(); + let mut scheduled = ScheduledNetworkWrites::new(64, 1024 * 1024); + scheduled + .push( + ScheduledNetworkClass::Priority, + now + Duration::from_millis(200), + scheduled_test_message(20), + ) + .unwrap(); + scheduled + .push( + ScheduledNetworkClass::Ordinary, + now + Duration::from_millis(100), + scheduled_test_message(10), + ) + .unwrap(); + assert_eq!( + scheduled_test_marker( + scheduled + .pop_ready(now + Duration::from_millis(100)) + .unwrap() + ), + 10 + ); + assert!( + scheduled + .pop_ready(now + Duration::from_millis(199)) + .is_none() + ); + assert_eq!( + scheduled_test_marker( + scheduled + .pop_ready(now + Duration::from_millis(200)) + .unwrap() + ), + 20 + ); + } + + #[test] + fn scheduled_writer_bounds_each_lane_without_cross_lane_eviction() { + let now = Instant::now(); + let one = scheduled_test_message(1); + let framed_bytes = one.len(); + let mut scheduled = ScheduledNetworkWrites::new(2, framed_bytes * 2); + for marker in [1, 2] { + scheduled + .push( + ScheduledNetworkClass::Ordinary, + now, + scheduled_test_message(marker), + ) + .unwrap(); + } + assert!( + scheduled + .push( + ScheduledNetworkClass::Ordinary, + now, + scheduled_test_message(3), + ) + .is_err() + ); + // Saturating the proactive/ordinary lane cannot consume priority + // count or byte credit. + for marker in [10, 11] { + scheduled + .push( + ScheduledNetworkClass::Priority, + now, + scheduled_test_message(marker), + ) + .unwrap(); + } + assert_eq!(scheduled.priority.writes.len(), 2); + assert_eq!(scheduled.ordinary.writes.len(), 2); + assert_eq!(scheduled.priority.bytes, framed_bytes * 2); + assert_eq!(scheduled.ordinary.bytes, framed_bytes * 2); + assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 10); + assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 11); + assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 1); + } + + #[tokio::test] + async fn connection_priority_channel_is_bounded_and_failure_is_observable() { + let (priority_sender, mut priority_receiver) = + mpsc::channel(RBC_DAG_PRIORITY_CHANNEL_CAPACITY); + for marker in 0..RBC_DAG_PRIORITY_CHANNEL_CAPACITY as RoundNumber { + priority_sender + .try_send(NetworkMessage::SubscribeBroadcastRequest(marker)) + .unwrap(); + } + assert!(matches!( + priority_sender.try_send(NetworkMessage::SubscribeBroadcastRequest(999)), + Err(mpsc::error::TrySendError::Full(_)) + )); + assert!(priority_receiver.recv().await.is_some()); + priority_sender + .try_send(NetworkMessage::SubscribeBroadcastRequest(999)) + .unwrap(); + + let (failure_sender, mut failure_receiver) = watch::channel(None); + failure_sender.send_replace(Some("scheduler saturated".to_owned())); + failure_receiver.changed().await.unwrap(); + assert_eq!( + failure_receiver.borrow().as_deref(), + Some("scheduler saturated") + ); + } + async fn connected_pair( addresses: &[SocketAddr; 2], parameters: &NodeParameters, @@ -1094,19 +1865,8 @@ mod tests { // the server tasks makes listener release deterministic for this test; // dropping their worker senders must then cancel every scoped stream // future and its OwnedWriteHalf. - network_0.abort_server(); - network_1.abort_server(); - let Network { - connection_receiver: connection_receiver_0, - server_task: server_task_0, - } = network_0; - let Network { - connection_receiver: connection_receiver_1, - server_task: server_task_1, - } = network_1; - drop(connection_receiver_0); - drop(connection_receiver_1); - let (result_0, result_1) = tokio::join!(server_task_0, server_task_1); + let (result_0, result_1) = + tokio::join!(network_0.abort_and_wait(), network_1.abort_and_wait()); assert!(result_0.is_err_and(|error| error.is_cancelled())); assert!(result_1.is_err_and(|error| error.is_cancelled())); } @@ -1170,9 +1930,13 @@ mod tests { )); let request = NetworkMessage::RbcHeaderRequest(block_ref); let response = NetworkMessage::RbcHeaderResponse(header); + let application_payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![0xAC; 8]), + )])); let shadow = NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { canonical_carrier: vec![0xA3, 0xA4], authentication_sidecar: vec![0xA5], + application_payload: Some(Arc::clone(&application_payload)), }); let shadow_request = NetworkMessage::RbcDagShadowCarrierRequest(block_ref); let shadow_response = @@ -1192,6 +1956,12 @@ mod tests { canonical_carrier: vec![0xA8, 0xA9], authentication_sidecar: vec![0xAA, 0xAB], }); + let payload_request = NetworkMessage::RbcDagApplicationPayloadRequest(block_ref); + let payload_response = + NetworkMessage::RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse { + application: block_ref, + transaction_data: application_payload, + }); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), @@ -1203,6 +1973,8 @@ mod tests { (shadow_response, 17, "rbc_dag_shadow_carrier_response"), (sync_request, 18, "rbc_dag_shadow_carrier_sync_request"), (sync_response, 19, "rbc_dag_shadow_carrier_sync_response"), + (payload_request, 20, "rbc_dag_application_payload_request"), + (payload_response, 21, "rbc_dag_application_payload_response"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); @@ -1244,6 +2016,83 @@ mod tests { )); } + #[test] + fn rbc_dag_application_payload_sidecars_roundtrip_exactly() { + let application = BlockReference::new_test(2, 17); + let transaction_data = Arc::new(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![0xE1, 0xE2, 0xE3]), + )])); + let carrier = RbcDagShadowCarrier { + canonical_carrier: vec![0xC1, 0xC2], + authentication_sidecar: vec![0xD1], + application_payload: Some(Arc::clone(&transaction_data)), + }; + let encoded = bincode::serialize(&NetworkMessage::RbcDagShadowCarrier(carrier)).unwrap(); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + let NetworkMessage::RbcDagShadowCarrier(decoded) = decoded else { + panic!("decoded a different network-message variant"); + }; + assert_eq!(decoded.canonical_carrier, vec![0xC1, 0xC2]); + assert_eq!(decoded.authentication_sidecar, vec![0xD1]); + let decoded_payload = decoded + .application_payload + .expect("application payload should survive the wire round trip"); + assert_eq!(decoded_payload.number_transactions(), 1); + let BaseTransaction::Share(transaction) = &decoded_payload.transactions()[0]; + assert_eq!(transaction.as_bytes(), &[0xE1, 0xE2, 0xE3]); + + let payloadless = RbcDagShadowCarrier { + canonical_carrier: vec![0xC3], + authentication_sidecar: vec![0xD2], + application_payload: None, + }; + let encoded = + bincode::serialize(&NetworkMessage::RbcDagShadowCarrier(payloadless)).unwrap(); + assert_eq!( + encoded, + vec![ + 15, 0, 0, 0, // frozen enum discriminant + 1, 0, 0, 0, 0, 0, 0, 0, 0xC3, // canonical carrier bytes + 1, 0, 0, 0, 0, 0, 0, 0, 0xD2, // authentication sidecar bytes + 0, // no application payload + ], + "payloadless carrier wire grammar changed", + ); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + assert!(matches!( + decoded, + NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { + application_payload: None, + .. + }) + )); + + let request = NetworkMessage::RbcDagApplicationPayloadRequest(application); + assert_eq!(variant_index(&request), 20); + let encoded = bincode::serialize(&request).unwrap(); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + assert!(matches!( + decoded, + NetworkMessage::RbcDagApplicationPayloadRequest(decoded) if decoded == application + )); + + let response = + NetworkMessage::RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse { + application, + transaction_data, + }); + assert_eq!(variant_index(&response), 21); + let encoded = bincode::serialize(&response).unwrap(); + let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); + let NetworkMessage::RbcDagApplicationPayloadResponse(decoded) = decoded else { + panic!("decoded a different network-message variant"); + }; + assert_eq!(decoded.application, application); + assert_eq!(decoded.transaction_data.number_transactions(), 1); + let BaseTransaction::Share(transaction) = &decoded.transaction_data.transactions()[0]; + assert_eq!(transaction.as_bytes(), &[0xE1, 0xE2, 0xE3]); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn scoped_connection_tasks_allow_immediate_same_port_rebind() { // Active sockets bind to listener_port * 10. Keep those derived ports diff --git a/crates/starfish-core/src/rocks_store.rs b/crates/starfish-core/src/rocks_store.rs index 6b8c040c..90c1d834 100644 --- a/crates/starfish-core/src/rocks_store.rs +++ b/crates/starfish-core/src/rocks_store.rs @@ -15,7 +15,7 @@ use crate::{ crypto::BlockDigest, dag_state::CommitData, data::Data, - store::Store, + store::{RbcDagFrontierReceipt, Store, validate_rbc_dag_frontier_commit_batch}, types::{ BlockHeader, BlockReference, ProvableShard, RoundNumber, TransactionData, VerifiedBlock, }, @@ -28,6 +28,8 @@ const CF_TX_DATA: &str = "tx_data"; const CF_SHARD_DATA: &str = "shard_data"; const CF_COMMITS: &str = "commits"; const CF_DUAL_DAG_CLEAN: &str = "sailfish_certified"; +const CF_RBC_DAG_FRONTIER_RECEIPT: &str = "rbc_dag_frontier_receipt"; +const LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY: &[u8] = b"latest"; pub struct RocksStore { db: Arc, @@ -144,6 +146,7 @@ impl RocksStore { ColumnFamilyDescriptor::new(CF_SHARD_DATA, Self::data_cf_options()), ColumnFamilyDescriptor::new(CF_COMMITS, Self::metadata_cf_options()), ColumnFamilyDescriptor::new(CF_DUAL_DAG_CLEAN, Self::metadata_cf_options()), + ColumnFamilyDescriptor::new(CF_RBC_DAG_FRONTIER_RECEIPT, Self::metadata_cf_options()), ]; let db = DB::open_cf_descriptors(&opts, path, cf_descriptors).map_err(io::Error::other)?; @@ -407,6 +410,38 @@ impl Store for RocksStore { .map_err(io::Error::other) } + fn store_commits_with_rbc_dag_receipt( + &self, + committed_sub_dags: Vec, + receipt: RbcDagFrontierReceipt, + ) -> io::Result<()> { + validate_rbc_dag_frontier_commit_batch(&committed_sub_dags, &receipt)?; + let receipt_bytes = receipt.to_bytes()?; + + let mut wb = rocksdb::WriteBatch::default(); + let cf_commits = self.cf(CF_COMMITS)?; + if committed_sub_dags.is_empty() { + let key = serialize(&receipt.carrier_anchor).map_err(io::Error::other)?; + wb.delete_cf(&cf_commits, key); + } else { + let commit_data = &committed_sub_dags[0]; + let key = serialize(&commit_data.leader).map_err(io::Error::other)?; + let value = serialize(commit_data).map_err(io::Error::other)?; + wb.put_cf(&cf_commits, key, value); + } + + let cf_receipt = self.cf(CF_RBC_DAG_FRONTIER_RECEIPT)?; + wb.put_cf( + &cf_receipt, + LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY, + receipt_bytes, + ); + + self.db + .write_opt(wb, &self.write_opts) + .map_err(io::Error::other) + } + fn get_commit(&self, reference: &BlockReference) -> io::Result> { let key = serialize(reference).map_err(io::Error::other)?; let cf_commits = self.cf(CF_COMMITS)?; @@ -423,6 +458,22 @@ impl Store for RocksStore { } } + fn read_latest_rbc_dag_frontier_receipt(&self) -> io::Result> { + let cf = self.cf(CF_RBC_DAG_FRONTIER_RECEIPT)?; + match self + .db + .get_cf_opt( + &cf, + LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY, + &Self::get_read_opts(), + ) + .map_err(io::Error::other)? + { + Some(bytes) => RbcDagFrontierReceipt::from_bytes(&bytes).map(Some), + None => Ok(None), + } + } + fn store_header_bytes(&self, reference: &BlockReference, bytes: &[u8]) -> io::Result<()> { let key = serialize(reference).map_err(io::Error::other)?; let cf = self.cf(CF_HEADERS)?; @@ -596,3 +647,158 @@ impl Store for RocksStore { Ok(refs) } } + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::RocksStore; + use crate::{ + dag_state::CommitData, + store::{RbcDagFrontierReceipt, Store}, + types::{BlockReference, MAX_COMMITTEE_SIZE}, + }; + + fn commit(leader: BlockReference, committed_rounds: Vec) -> CommitData { + CommitData { + leader, + sub_dag: vec![BlockReference::new_test(1, leader.round)], + committed_rounds, + } + } + + fn assert_commit(store: &impl Store, expected: &CommitData) { + let actual = store + .get_commit(&expected.leader) + .expect("commit read should succeed") + .expect("commit should exist"); + assert_eq!(actual.leader, expected.leader); + assert_eq!(actual.sub_dag, expected.sub_dag); + assert_eq!(actual.committed_rounds, expected.committed_rounds); + } + + #[test] + fn rbc_dag_receipt_and_commits_are_atomic_and_latest_is_a_point_value() { + let temp_dir = TempDir::new().unwrap(); + let store = RocksStore::open(temp_dir.path()).unwrap(); + + let legacy_leader = BlockReference::new_test(2, 253); + let legacy_commit = commit(legacy_leader, vec![253; 4]); + store.store_commits(vec![legacy_commit.clone()]).unwrap(); + assert_commit(&store, &legacy_commit); + assert!( + store + .read_latest_rbc_dag_frontier_receipt() + .unwrap() + .is_none() + ); + + // A control-only frontier has no new application commits, but its + // durable cursor must still advance. + let first_anchor = BlockReference::new_test(7, 255); + let first_receipt = RbcDagFrontierReceipt { + carrier_anchor: first_anchor, + output_sequence: 255, + committed_rounds: vec![250, 251, 252, 253], + }; + let stale_first_commit = commit(first_anchor, first_receipt.committed_rounds.clone()); + store.store_commits(vec![stale_first_commit]).unwrap(); + assert!(store.get_commit(&first_anchor).unwrap().is_some()); + store + .store_commits_with_rbc_dag_receipt(Vec::new(), first_receipt.clone()) + .unwrap(); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(first_receipt) + ); + assert!(store.get_commit(&first_anchor).unwrap().is_none()); + + // The exact application commit is stored under the consensus carrier + // anchor so Core can reconstruct the compact receipt's application + // references after restart. + let second_anchor = BlockReference::new_test(7, 256); + let application_commit = commit(second_anchor, vec![255, 256, 255, 256]); + let second_receipt = RbcDagFrontierReceipt { + carrier_anchor: second_anchor, + output_sequence: 256, + committed_rounds: vec![255, 256, 255, 256], + }; + store + .store_commits_with_rbc_dag_receipt( + vec![application_commit.clone()], + second_receipt.clone(), + ) + .unwrap(); + assert_commit(&store, &application_commit); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(second_receipt.clone()) + ); + + // Mismatched/multiple application commits are rejected before either + // commit data or the latest receipt can change. + let mismatched = commit(BlockReference::new_test(2, 254), vec![255, 256, 255, 256]); + let mismatched_watermarks = commit(second_anchor, vec![1; 4]); + for invalid in [ + vec![mismatched], + vec![mismatched_watermarks], + vec![application_commit.clone(), application_commit.clone()], + ] { + let error = store + .store_commits_with_rbc_dag_receipt(invalid, second_receipt.clone()) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(second_receipt.clone()) + ); + } + + // Reusing the exact anchor for a control-only marker atomically + // removes stale application CommitData, preserving absence semantics. + let control_receipt = RbcDagFrontierReceipt { + carrier_anchor: second_anchor, + output_sequence: 257, + committed_rounds: second_receipt.committed_rounds.clone(), + }; + store + .store_commits_with_rbc_dag_receipt(Vec::new(), control_receipt.clone()) + .unwrap(); + assert!(store.get_commit(&second_anchor).unwrap().is_none()); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(control_receipt.clone()) + ); + + // Receipt validation happens before the batch is submitted, so an + // invalid vector cannot partially write its application commit or + // replace the last valid cursor. + let rejected_leader = BlockReference::new_test(3, 257); + let rejected = commit(rejected_leader, vec![257; 4]); + for committed_rounds in [Vec::new(), vec![0; usize::from(MAX_COMMITTEE_SIZE) + 1]] { + let invalid = RbcDagFrontierReceipt { + carrier_anchor: BlockReference::new_test(7, 257), + output_sequence: 258, + committed_rounds, + }; + let error = store + .store_commits_with_rbc_dag_receipt(vec![rejected.clone()], invalid) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + } + assert!(store.get_commit(&rejected_leader).unwrap().is_none()); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(control_receipt.clone()) + ); + + drop(store); + let reopened = RocksStore::open(temp_dir.path()).unwrap(); + assert_commit(&reopened, &legacy_commit); + assert!(reopened.get_commit(&second_anchor).unwrap().is_none()); + assert_eq!( + reopened.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(control_receipt) + ); + } +} diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index a68f2fe7..eddbb10a 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -8,7 +8,12 @@ //! decoding. Callers validate canonical bytes before journaling them; the //! reducer pins the exact byte strings and rejects any later alternative. -use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, + sync::Arc, +}; use crate::types::{AuthorityIndex, BlockReference, RoundNumber}; @@ -107,6 +112,8 @@ impl DurableOutboundCarrierV1 { pub enum PhaseKindV1 { Echo, Ready, + Vote, + Ack, } /// Durable result of processing one entry in an enclosing phase batch. @@ -122,6 +129,8 @@ impl PhaseKindV1 { match statement { RbcPhaseStatementV1::Echo { .. } => Self::Echo, RbcPhaseStatementV1::Ready { .. } => Self::Ready, + RbcPhaseStatementV1::Vote { .. } => Self::Vote, + RbcPhaseStatementV1::Ack { .. } => Self::Ack, } } } @@ -174,6 +183,20 @@ pub enum JournalEventV1 { context: RbcDagContextV1, target: BlockReference, }, + LockVote { + context: RbcDagContextV1, + target: BlockReference, + }, + LockAck { + context: RbcDagContextV1, + target: BlockReference, + }, + /// The sender-honest or optimistic-ECHO fast-delivery predicate became + /// slot-global before the slower Q-READY certificate. + LockOptimisticDelivery { + context: RbcDagContextV1, + target: BlockReference, + }, LockDelivery { context: RbcDagContextV1, target: BlockReference, @@ -223,6 +246,9 @@ impl JournalEventV1 { | Self::LockEcho { context, .. } | Self::LockAdmission { context, .. } | Self::LockReady { context, .. } + | Self::LockVote { context, .. } + | Self::LockAck { context, .. } + | Self::LockOptimisticDelivery { context, .. } | Self::LockDelivery { context, .. } | Self::LockConsensusSlot { context, .. } | Self::LockLeaderChoice { context, .. } @@ -277,11 +303,18 @@ pub struct JournalSnapshotV1 { context: RbcDagContextV1, own_authority: AuthorityIndex, ingress: Vec, + /// Derived exact index over `ingress`. This is intentionally absent from + /// the durable format: replay reconstructs it while preserving the + /// ordered ingress records above. + authenticated_ingress_references: BTreeSet, retained_carriers: BTreeMap, own_carriers: BTreeMap, admission_locks: BTreeMap, echo_locks: BTreeMap, ready_locks: BTreeMap, + vote_locks: BTreeMap, + ack_locks: BTreeMap, + optimistic_delivery_locks: BTreeMap, delivery_locks: BTreeMap, consensus_slots: BTreeMap, leader_choices: BTreeMap, @@ -297,11 +330,15 @@ impl JournalSnapshotV1 { context, own_authority, ingress: Vec::new(), + authenticated_ingress_references: BTreeSet::new(), retained_carriers: BTreeMap::new(), own_carriers: BTreeMap::new(), admission_locks: BTreeMap::new(), echo_locks: BTreeMap::new(), ready_locks: BTreeMap::new(), + vote_locks: BTreeMap::new(), + ack_locks: BTreeMap::new(), + optimistic_delivery_locks: BTreeMap::new(), delivery_locks: BTreeMap::new(), consensus_slots: BTreeMap::new(), leader_choices: BTreeMap::new(), @@ -350,10 +387,22 @@ impl JournalSnapshotV1 { self.ready_locks.get(&slot).copied() } + pub fn vote_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.vote_locks.get(&slot).copied() + } + + pub fn ack_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.ack_locks.get(&slot).copied() + } + pub fn delivery_lock(&self, slot: RbcSlotKeyV1) -> Option { self.delivery_locks.get(&slot).copied() } + pub fn optimistic_delivery_lock(&self, slot: RbcSlotKeyV1) -> Option { + self.optimistic_delivery_locks.get(&slot).copied() + } + pub fn consensus_slot(&self, round: RoundNumber) -> Option { self.consensus_slots.get(&round).copied() } @@ -414,11 +463,41 @@ impl JournalSnapshotV1 { JournalEventV1::LockAdmission { target, .. } => self.lock_admission(*target), JournalEventV1::LockEcho { target, .. } => self.lock_echo(*target), JournalEventV1::LockReady { target, .. } => self.lock_ready(*target), + JournalEventV1::LockVote { target, .. } => self.lock_vote(*target), + JournalEventV1::LockAck { target, .. } => self.lock_ack(*target), + JournalEventV1::LockOptimisticDelivery { target, .. } => { + self.ensure_retained(*target)?; + let slot = RbcSlotKeyV1::of(*target); + if self + .delivery_lock(slot) + .is_some_and(|certified| certified != *target) + { + return Err(JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::OptimisticDelivery, + slot, + }); + } + Self::lock_candidate( + &mut self.optimistic_delivery_locks, + *target, + LockKindV1::OptimisticDelivery, + ) + } JournalEventV1::LockDelivery { target, .. } => { self.ensure_retained(*target)?; - if self.ready_lock(RbcSlotKeyV1::of(*target)) != Some(*target) { + let slot = RbcSlotKeyV1::of(*target); + if self.ready_lock(slot) != Some(*target) { return Err(JournalErrorV1::DeliveryWithoutMatchingReady(*target)); } + if self + .optimistic_delivery_lock(slot) + .is_some_and(|delivered| delivered != *target) + { + return Err(JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Delivery, + slot, + }); + } Self::lock_candidate(&mut self.delivery_locks, *target, LockKindV1::Delivery) } JournalEventV1::LockConsensusSlot { @@ -483,6 +562,7 @@ impl JournalSnapshotV1 { canonical_carrier_wire, authentication_sidecar, }); + self.authenticated_ingress_references.insert(reference); Ok(()) } @@ -518,11 +598,7 @@ impl JournalSnapshotV1 { } fn lock_admission(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - let authenticated_ingress = self - .ingress - .iter() - .any(|ingress| ingress.reference == target); - if !authenticated_ingress { + if !self.authenticated_ingress_references.contains(&target) { return Err(JournalErrorV1::AdmissionWithoutAuthenticatedIngress(target)); } Self::lock_candidate(&mut self.admission_locks, target, LockKindV1::Admission) @@ -533,6 +609,16 @@ impl JournalSnapshotV1 { Self::lock_candidate(&mut self.ready_locks, target, LockKindV1::Ready) } + fn lock_vote(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { + self.ensure_retained(target)?; + Self::lock_candidate(&mut self.vote_locks, target, LockKindV1::Vote) + } + + fn lock_ack(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { + self.ensure_retained(target)?; + Self::lock_candidate(&mut self.ack_locks, target, LockKindV1::Ack) + } + fn ensure_retained(&self, reference: BlockReference) -> Result<(), JournalErrorV1> { if self.retained_carriers.contains_key(&reference) { Ok(()) @@ -740,14 +826,17 @@ impl JournalSnapshotV1 { return Err(JournalErrorV1::OutboundSidecarNotPersisted(reference)); } let candidate = outbound.candidate.clone(); - if self.echo_lock(RbcSlotKeyV1::of(reference)) != Some(reference) { - return Err(JournalErrorV1::OutboundEchoNotLocked(reference)); - } + // The target author is excluded from ECHO/VOTE/ACK in the optimistic + // RBC. `FixOwnCarrier` is the durable authority lock for exposing its + // own exact carrier; only phase statements carried inside it require + // their corresponding local locks below. for statement in candidate.header().phase_batch() { let target = statement.target(); let lock = match statement { RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), + RbcPhaseStatementV1::Vote { .. } => self.vote_lock(RbcSlotKeyV1::of(target)), + RbcPhaseStatementV1::Ack { .. } => self.ack_lock(RbcSlotKeyV1::of(target)), }; if lock != Some(target) { return Err(JournalErrorV1::OutboundPhaseNotLocked(*statement)); @@ -807,6 +896,7 @@ impl JournalSnapshotV1 { let outer_slot = RbcSlotKeyV1::of(outer); let authorized = self.own_carrier(outer.round) == Some(outer) || self.admission_lock(outer_slot) == Some(outer) + || self.optimistic_delivery_lock(outer_slot) == Some(outer) || self.delivery_lock(outer_slot) == Some(outer); if !authorized { return Err(JournalErrorV1::OuterCarrierNotAdmittedOrDelivered(outer)); @@ -838,6 +928,8 @@ impl JournalSnapshotV1 { let lock = match statement { RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), + RbcPhaseStatementV1::Vote { .. } => self.vote_lock(RbcSlotKeyV1::of(target)), + RbcPhaseStatementV1::Ack { .. } => self.ack_lock(RbcSlotKeyV1::of(target)), }; if lock != Some(target) { return Err(JournalErrorV1::OwnPhaseWithoutDurableLock(statement)); @@ -921,6 +1013,9 @@ pub enum LockKindV1 { Admission, Echo, Ready, + Vote, + Ack, + OptimisticDelivery, Delivery, } @@ -970,7 +1065,6 @@ pub enum JournalErrorV1 { OutboundSidecarNotPersisted(BlockReference), OutboundAuthenticationContextMismatch, OutboundAuthenticationCandidateMismatch(BlockReference), - OutboundEchoNotLocked(BlockReference), OutboundPhaseNotLocked(RbcPhaseStatementV1), OutboundConsensusSlotNotLocked { consensus_round: RoundNumber, @@ -1394,6 +1488,26 @@ mod tests { .unwrap(); } + fn assert_authenticated_ingress_index_matches_scan(snapshot: &JournalSnapshotV1) { + let scanned = snapshot + .authenticated_ingress() + .iter() + .map(AuthenticatedIngressRecordV1::reference) + .collect::>(); + assert_eq!(snapshot.authenticated_ingress_references, scanned); + for reference in scanned { + assert_eq!( + snapshot + .authenticated_ingress_references + .contains(&reference), + snapshot + .authenticated_ingress() + .iter() + .any(|record| record.reference() == reference) + ); + } + } + fn admit_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { authenticate_candidate(journal, candidate); journal @@ -1609,6 +1723,211 @@ mod tests { )); } + #[test] + fn authenticated_ingress_index_matches_scan_after_live_failures_and_duplicates() { + let mut journal = journal(); + let candidate = candidate(0, 1, 0x22, Vec::new(), None); + let reference = candidate.reference(); + + assert_eq!( + journal + .append(JournalEventV1::AuthenticatedIngress { + context: journal.context, + sequence: 0, + authenticated: authenticated_with_marker(&candidate, 1, 0xA2), + provenance: IngressProvenanceV1::Relayed { peer: 2 }, + }) + .unwrap_err(), + JournalErrorV1::AuthenticatedIngressContextMismatch + ); + assert_authenticated_ingress_index_matches_scan(journal.snapshot()); + assert!( + !journal + .snapshot() + .authenticated_ingress_references + .contains(&reference) + ); + + let authenticated = authenticated(&candidate, 1); + journal + .record_authenticated_ingress( + authenticated.clone(), + IngressProvenanceV1::Relayed { peer: 2 }, + ) + .unwrap(); + journal + .record_authenticated_ingress(authenticated, IngressProvenanceV1::Relayed { peer: 3 }) + .unwrap(); + + assert_eq!(journal.snapshot().authenticated_ingress().len(), 2); + assert_eq!(journal.snapshot().authenticated_ingress_references.len(), 1); + assert_authenticated_ingress_index_matches_scan(journal.snapshot()); + } + + #[test] + fn authenticated_ingress_index_reconstructs_a_long_ordered_sequence() { + const REPEATED_INGRESS: usize = 512; + const UNIQUE_INGRESS: usize = 16; + + let context = context(0xA1); + let mut durable_events = Vec::with_capacity(REPEATED_INGRESS + UNIQUE_INGRESS); + let mut ordered_references = Vec::with_capacity(REPEATED_INGRESS + UNIQUE_INGRESS); + let mut unique_references = Vec::with_capacity(UNIQUE_INGRESS); + let first_candidate = candidate(0, 1, 0x26, Vec::new(), None); + let first_reference = first_candidate.reference(); + let first_authenticated = authenticated(&first_candidate, 1); + unique_references.push(first_reference); + for index in 0..REPEATED_INGRESS { + ordered_references.push(first_reference); + durable_events.push(JournalEventV1::AuthenticatedIngress { + context, + sequence: durable_events.len() as u64, + authenticated: first_authenticated.clone(), + provenance: IngressProvenanceV1::Relayed { + peer: if index % 2 == 0 { 2 } else { 3 }, + }, + }); + } + for index in 1..UNIQUE_INGRESS { + let candidate = candidate( + 0, + RoundNumber::try_from(index + 1).unwrap(), + (index as u8).wrapping_mul(37), + Vec::new(), + None, + ); + let reference = candidate.reference(); + let authenticated = authenticated(&candidate, 1); + unique_references.push(reference); + ordered_references.push(reference); + durable_events.push(JournalEventV1::AuthenticatedIngress { + context, + sequence: durable_events.len() as u64, + authenticated, + provenance: IngressProvenanceV1::Relayed { peer: 2 }, + }); + } + + let ingress_events = durable_events.clone(); + let mut journal = + WriteAheadJournalV1::from_durable_events(context, 1, durable_events).unwrap(); + assert_eq!(journal.durable_events(), ingress_events); + assert_eq!( + journal + .snapshot() + .authenticated_ingress() + .iter() + .map(AuthenticatedIngressRecordV1::reference) + .collect::>(), + ordered_references + ); + assert_eq!( + journal.snapshot().authenticated_ingress_references.len(), + UNIQUE_INGRESS + ); + assert_authenticated_ingress_index_matches_scan(journal.snapshot()); + + for target in [ + unique_references[0], + unique_references[UNIQUE_INGRESS / 2], + unique_references[UNIQUE_INGRESS - 1], + ] { + journal + .append(JournalEventV1::LockAdmission { context, target }) + .unwrap(); + } + + let reopened = journal.restart().unwrap().restart().unwrap(); + assert_authenticated_ingress_index_matches_scan(reopened.snapshot()); + assert_eq!(reopened.snapshot(), journal.snapshot()); + } + + #[test] + fn authenticated_ingress_index_preserves_admission_conflicts_across_reopen() { + let mut journal = journal(); + let first_candidate = candidate(0, 7, 0x23, Vec::new(), None); + let conflicting_candidate = candidate(0, 7, 0x24, Vec::new(), None); + let absent_candidate = candidate(2, 9, 0x25, Vec::new(), None); + let first = first_candidate.reference(); + let conflicting = conflicting_candidate.reference(); + let absent = absent_candidate.reference(); + let slot = RbcSlotKeyV1::of(first); + + authenticate_candidate(&mut journal, &first_candidate); + authenticate_candidate(&mut journal, &conflicting_candidate); + assert_authenticated_ingress_index_matches_scan(journal.snapshot()); + assert!( + journal + .snapshot() + .authenticated_ingress_references + .contains(&first) + ); + assert!( + journal + .snapshot() + .authenticated_ingress_references + .contains(&conflicting) + ); + assert!( + !journal + .snapshot() + .authenticated_ingress_references + .contains(&absent) + ); + + journal + .append(JournalEventV1::LockAdmission { + context: journal.context, + target: first, + }) + .unwrap(); + assert_eq!( + journal + .append(JournalEventV1::LockAdmission { + context: journal.context, + target: conflicting, + }) + .unwrap_err(), + JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Admission, + slot, + } + ); + assert_eq!( + journal + .append(JournalEventV1::LockAdmission { + context: journal.context, + target: absent, + }) + .unwrap_err(), + JournalErrorV1::AdmissionWithoutAuthenticatedIngress(absent) + ); + + let mut reopened = journal.restart().unwrap(); + assert_authenticated_ingress_index_matches_scan(reopened.snapshot()); + assert_eq!( + reopened + .append(JournalEventV1::LockAdmission { + context: reopened.context, + target: conflicting, + }) + .unwrap_err(), + JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Admission, + slot, + } + ); + assert_eq!( + reopened + .append(JournalEventV1::LockAdmission { + context: reopened.context, + target: absent, + }) + .unwrap_err(), + JournalErrorV1::AdmissionWithoutAuthenticatedIngress(absent) + ); + } + #[test] fn crash_boundaries_preserve_each_slot_global_lock() { let own_candidate = candidate(1, 1, 0x11, Vec::new(), None); @@ -1825,7 +2144,7 @@ mod tests { } #[test] - fn only_admitted_conflict_processes_until_the_other_is_delivered() { + fn optimistic_delivery_authorizes_unadmitted_outer_batch_and_survives_q_ready() { let mut journal = journal(); let first_statement = RbcPhaseStatementV1::Echo { target: reference(0, 1, 0x63), @@ -1878,13 +2197,7 @@ mod tests { ); journal - .append(JournalEventV1::LockReady { - context: journal.context, - target: second, - }) - .unwrap(); - journal - .append(JournalEventV1::LockDelivery { + .append(JournalEventV1::LockOptimisticDelivery { context: journal.context, target: second, }) @@ -1906,6 +2219,33 @@ mod tests { }) .unwrap(); assert_eq!(journal.snapshot().phase_batch_cursor(second), 1); + assert_eq!( + journal + .restart() + .unwrap() + .snapshot() + .optimistic_delivery_lock(RbcSlotKeyV1::of(second)), + Some(second) + ); + + // The independent fallback certificate remains durable and may + // arrive after the fast-delivery latch without changing its value. + journal + .append(JournalEventV1::LockReady { + context: journal.context, + target: second, + }) + .unwrap(); + journal + .append(JournalEventV1::LockDelivery { + context: journal.context, + target: second, + }) + .unwrap(); + assert_eq!( + journal.snapshot().delivery_lock(RbcSlotKeyV1::of(second)), + Some(second) + ); } #[test] @@ -2087,6 +2427,192 @@ mod tests { ); } + #[test] + fn vote_and_ack_local_locks_are_phase_separate_slot_global_and_restart_safe() { + let mut journal = journal(); + let first_candidate = candidate(0, 1, 0xA4, Vec::new(), None); + let second_candidate = candidate(0, 1, 0xA5, Vec::new(), None); + let first = first_candidate.reference(); + let second = second_candidate.reference(); + let slot = RbcSlotKeyV1::of(first); + + assert_eq!( + journal + .append(JournalEventV1::LockVote { + context: journal.context, + target: first, + }) + .unwrap_err(), + JournalErrorV1::CarrierContentNotRetained(first) + ); + assert_eq!( + journal + .append(JournalEventV1::LockAck { + context: journal.context, + target: second, + }) + .unwrap_err(), + JournalErrorV1::CarrierContentNotRetained(second) + ); + + retain_candidate(&mut journal, &first_candidate); + retain_candidate(&mut journal, &second_candidate); + journal + .append(JournalEventV1::LockVote { + context: journal.context, + target: first, + }) + .unwrap(); + // ACK has its own local phase namespace; it need not match the local + // VOTE when independently sufficient evidence selects another value. + journal + .append(JournalEventV1::LockAck { + context: journal.context, + target: second, + }) + .unwrap(); + + assert!(matches!( + journal.append(JournalEventV1::LockVote { + context: journal.context, + target: second, + }), + Err(JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Vote, + slot: conflict_slot, + }) if conflict_slot == slot + )); + assert!(matches!( + journal.append(JournalEventV1::LockAck { + context: journal.context, + target: first, + }), + Err(JournalErrorV1::ConflictingPhaseLock { + kind: LockKindV1::Ack, + slot: conflict_slot, + }) if conflict_slot == slot + )); + + let restarted = journal.restart().unwrap().restart().unwrap(); + assert_eq!(restarted.snapshot().vote_lock(slot), Some(first)); + assert_eq!(restarted.snapshot().ack_lock(slot), Some(second)); + } + + #[test] + fn vote_and_ack_batches_durably_classify_counted_replay_and_equivocation() { + let mut journal = journal(); + let first = reference(0, 1, 0xA6); + let second = reference(0, 1, 0xA7); + let first_batch = [ + RbcPhaseStatementV1::Vote { target: first }, + RbcPhaseStatementV1::Ack { target: first }, + ]; + let replay_batch = first_batch; + let conflicting_batch = [ + RbcPhaseStatementV1::Vote { target: second }, + RbcPhaseStatementV1::Ack { target: second }, + ]; + let first_outer_candidate = candidate(2, 2, 0xA8, first_batch.to_vec(), None); + let replay_outer_candidate = candidate(2, 3, 0xA9, replay_batch.to_vec(), None); + let conflicting_outer_candidate = candidate(2, 4, 0xAA, conflicting_batch.to_vec(), None); + let first_outer = first_outer_candidate.reference(); + let replay_outer = replay_outer_candidate.reference(); + let conflicting_outer = conflicting_outer_candidate.reference(); + admit_candidate(&mut journal, &first_outer_candidate); + admit_candidate(&mut journal, &replay_outer_candidate); + admit_candidate(&mut journal, &conflicting_outer_candidate); + + // The journal binds each event to the exact candidate batch position; + // a valid statement from the wrong position cannot be applied. + assert_eq!( + journal + .append(JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: first_outer, + index: 0, + sender: 2, + statement: first_batch[1], + }) + .unwrap_err(), + JournalErrorV1::PhaseBatchEntryMismatch { + outer: first_outer, + index: 0, + } + ); + + let apply_batch = |outer, statements: [RbcPhaseStatementV1; 2], context| { + vec![ + JournalEventV1::ApplyPhaseStatement { + context, + outer, + index: 0, + sender: 2, + statement: statements[0], + }, + JournalEventV1::AdvancePhaseBatchCursor { + context, + outer, + index: 0, + }, + JournalEventV1::ApplyPhaseStatement { + context, + outer, + index: 1, + sender: 2, + statement: statements[1], + }, + JournalEventV1::AdvancePhaseBatchCursor { + context, + outer, + index: 1, + }, + ] + }; + for (outer, statements) in [ + (first_outer, first_batch), + (replay_outer, replay_batch), + (conflicting_outer, conflicting_batch), + ] { + let batch = journal + .validate_batch(apply_batch(outer, statements, journal.context)) + .unwrap(); + journal.commit_validated_batch(batch).unwrap(); + } + + for index in 0..2 { + assert_eq!( + journal + .snapshot() + .phase_statement_outcome(first_outer, index), + Some(AppliedPhaseOutcomeV1::Counted) + ); + assert_eq!( + journal + .snapshot() + .phase_statement_outcome(replay_outer, index), + Some(AppliedPhaseOutcomeV1::IgnoredReplay) + ); + assert_eq!( + journal + .snapshot() + .phase_statement_outcome(conflicting_outer, index), + Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) + ); + } + assert_eq!(journal.snapshot().phase_batch_cursor(first_outer), 2); + assert_eq!(journal.snapshot().phase_batch_cursor(replay_outer), 2); + assert_eq!(journal.snapshot().phase_batch_cursor(conflicting_outer), 2); + + let restarted = journal.restart().unwrap(); + assert_eq!(restarted.snapshot(), journal.snapshot()); + assert_eq!( + restarted + .snapshot() + .phase_statement_outcome(conflicting_outer, 1), + Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) + ); + } + #[test] fn replay_is_idempotent_and_foreign_namespace_fails_closed() { let mut journal = journal(); @@ -2227,6 +2753,90 @@ mod tests { journal.append(expose).unwrap(); } + #[test] + fn outbound_exposure_waits_for_vote_and_ack_locks_in_exact_batch_order() { + let mut journal = journal(); + let vote_target_candidate = candidate(0, 1, 0xB8, Vec::new(), None); + let ack_target_candidate = candidate(2, 1, 0xB9, Vec::new(), None); + let vote = RbcPhaseStatementV1::Vote { + target: vote_target_candidate.reference(), + }; + let ack = RbcPhaseStatementV1::Ack { + target: ack_target_candidate.reference(), + }; + let own_candidate = candidate(1, 2, 0xBA, vec![vote, ack], None); + let own = own_candidate.reference(); + retain_candidate(&mut journal, &vote_target_candidate); + retain_candidate(&mut journal, &ack_target_candidate); + prepare_outbound_for_exposure(&mut journal, &own_candidate); + let expose = JournalEventV1::ExposeOutbound { + context: journal.context, + reference: own, + }; + + assert_eq!( + journal.append(expose.clone()).unwrap_err(), + JournalErrorV1::OutboundPhaseNotLocked(vote) + ); + let apply_vote = JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: own, + index: 0, + sender: 1, + statement: vote, + }; + assert_eq!( + journal.append(apply_vote.clone()).unwrap_err(), + JournalErrorV1::OwnPhaseWithoutDurableLock(vote) + ); + journal + .append(JournalEventV1::LockVote { + context: journal.context, + target: vote.target(), + }) + .unwrap(); + journal.append(apply_vote).unwrap(); + journal + .append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer: own, + index: 0, + }) + .unwrap(); + assert_eq!( + journal.append(expose.clone()).unwrap_err(), + JournalErrorV1::OutboundPhaseNotLocked(ack) + ); + let apply_ack = JournalEventV1::ApplyPhaseStatement { + context: journal.context, + outer: own, + index: 1, + sender: 1, + statement: ack, + }; + assert_eq!( + journal.append(apply_ack.clone()).unwrap_err(), + JournalErrorV1::OwnPhaseWithoutDurableLock(ack) + ); + journal + .append(JournalEventV1::LockAck { + context: journal.context, + target: ack.target(), + }) + .unwrap(); + journal.append(apply_ack).unwrap(); + journal + .append(JournalEventV1::AdvancePhaseBatchCursor { + context: journal.context, + outer: own, + index: 1, + }) + .unwrap(); + journal.append(expose).unwrap(); + assert!(journal.snapshot().outbound(own).unwrap().exposed()); + assert_eq!(journal.snapshot().phase_batch_cursor(own), 2); + } + #[test] fn outbound_exposure_waits_for_matching_consensus_and_leader_locks() { let mut journal = journal(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs index f734e527..092d8c60 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -63,14 +63,18 @@ const OPTION_NONE: u8 = 0; const OPTION_SOME: u8 = 1; const PHASE_ECHO: u8 = 0; const PHASE_READY: u8 = 1; +// Phase codes are append-only: persisted carrier headers and their digests +// depend on the original ECHO=0/READY=1 assignments. +const PHASE_VOTE: u8 = 2; +const PHASE_ACK: u8 = 3; const LEADER_NONE: u8 = 0; const LEADER_VOTE: u8 = 1; const LEADER_NO_VOTE: u8 = 2; const BLOCK_REFERENCE_SIZE: usize = 2 + 4 + 32; const PROTOCOL_INSTANCE_SIZE: usize = 32; const COMMITTEE_ID_SIZE: usize = 32; -const AUTHENTICATION_DOMAIN: &[u8; 19] = b"STARFISH_RBC_DAG_V1"; -const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_DAG_V1_COMMITTEE_ID"; +const AUTHENTICATION_DOMAIN: &[u8; 19] = b"STARFISH_RBC_DAG_V2"; +const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_DAG_V2_COMMITTEE_ID"; const CARRIER_AUTHENTICATION_KIND: u8 = 0; const AUTHENTICATION_BASE_SIZE: usize = 123; const AUTHENTICATION_MAC_SIZE: usize = AUTHENTICATION_BASE_SIZE + 2; @@ -84,12 +88,17 @@ std::thread_local! { pub enum RbcPhaseStatementV1 { Echo { target: BlockReference }, Ready { target: BlockReference }, + Vote { target: BlockReference }, + Ack { target: BlockReference }, } impl RbcPhaseStatementV1 { pub fn target(self) -> BlockReference { match self { - Self::Echo { target } | Self::Ready { target } => target, + Self::Echo { target } + | Self::Ready { target } + | Self::Vote { target } + | Self::Ack { target } => target, } } @@ -97,6 +106,8 @@ impl RbcPhaseStatementV1 { match self { Self::Echo { .. } => PHASE_ECHO, Self::Ready { .. } => PHASE_READY, + Self::Vote { .. } => PHASE_VOTE, + Self::Ack { .. } => PHASE_ACK, } } } @@ -1697,7 +1708,11 @@ fn validate_outer_header( } } - let phase_limit = usize::min(MAX_PHASE_STATEMENTS_V1, committee.len().saturating_mul(4)); + // Four protocol phases plus bounded spillover from prior carrier rounds. + // Six entries per authority keeps the honest four-phase fast path away + // from the canonical validation boundary without making the wire vector + // unbounded. + let phase_limit = usize::min(MAX_PHASE_STATEMENTS_V1, committee.len().saturating_mul(6)); if header.phase_batch.len() > phase_limit { return Err(RbcDagError::VectorTooLong { field: "phase statements", @@ -2057,6 +2072,8 @@ fn decode_header( phase_batch.push(match code { PHASE_ECHO => RbcPhaseStatementV1::Echo { target }, PHASE_READY => RbcPhaseStatementV1::Ready { target }, + PHASE_VOTE => RbcPhaseStatementV1::Vote { target }, + PHASE_ACK => RbcPhaseStatementV1::Ack { target }, other => return Err(RbcDagError::InvalidPhase(other)), }); } @@ -2689,6 +2706,63 @@ mod tests { )); } + #[test] + fn vote_and_ack_phase_codes_are_append_only_and_round_trip() { + let committee = Committee::new_test(vec![1; 4]); + let target = reference(2, 1, 0xA8); + assert_eq!(RbcPhaseStatementV1::Echo { target }.code(), 0); + assert_eq!(RbcPhaseStatementV1::Ready { target }.code(), 1); + assert_eq!(RbcPhaseStatementV1::Vote { target }.code(), 2); + assert_eq!(RbcPhaseStatementV1::Ack { target }.code(), 3); + + let mut args = args(&committee, 3, 2); + args.phase_batch = vec![ + RbcPhaseStatementV1::Vote { target }, + RbcPhaseStatementV1::Ack { target }, + ]; + let candidate = CandidateCarrierV1::try_new(args, &committee).unwrap(); + let content = candidate.canonical_content_bytes().unwrap(); + let wire = candidate.canonical_wire_bytes().unwrap(); + assert_eq!( + CandidateCarrierV1::decode_content(&content, &committee, Some(candidate.reference())) + .unwrap(), + candidate + ); + assert_eq!( + CandidateCarrierV1::decode_wire(&wire, &committee, Some(candidate.reference())) + .unwrap(), + candidate + ); + } + + #[test] + fn four_phase_batch_accepts_six_entries_per_authority_at_the_boundary() { + let committee = Committee::new_test(vec![1; 4]); + let mut args = args(&committee, 3, 8); + for round in 1..=6 { + let target = reference(0, round, round as u8); + args.phase_batch.extend([ + RbcPhaseStatementV1::Echo { target }, + RbcPhaseStatementV1::Vote { target }, + RbcPhaseStatementV1::Ack { target }, + RbcPhaseStatementV1::Ready { target }, + ]); + } + assert_eq!(args.phase_batch.len(), committee.len() * 6); + CandidateCarrierV1::try_new(args.clone(), &committee).unwrap(); + + args.phase_batch.push(RbcPhaseStatementV1::Echo { + target: reference(1, 1, 0xFF), + }); + assert!(matches!( + CandidateCarrierV1::try_new(args, &committee), + Err(RbcDagError::VectorTooLong { + field: "phase statements", + count: 25, + }) + )); + } + #[test] fn acknowledgment_compression_normalizes_and_rejects_duplicates() { let committee = Committee::new_test(vec![1; 4]); @@ -2792,21 +2866,21 @@ mod tests { let base = context.public_authentication_statement(candidate.reference()); assert_eq!( hex::encode(context.committee_id().as_bytes()), - "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62" + "82804ac9a25b89ad8098c52ec0ec7cfe4250d23d672a627bdb5795eda8ec4b98" ); assert_eq!( hex::encode(base), concat!( - "53544152464953485f5242435f4441475f56310002", + "53544152464953485f5242435f4441475f56320002", "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", - "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", + "82804ac9a25b89ad8098c52ec0ec7cfe4250d23d672a627bdb5795eda8ec4b98", "000300000002", "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f" ) ); assert_eq!( hex::encode(context.public_authentication_digest(candidate.reference())), - "26a0866c9c6938c9158495f23fd22b281db6371461b6aad109ad41329e5fa5c8" + "79a89fb36b517a157591fa0d44755211c5dfccb604e10d0d52ee921a4e5dae48" ); assert_eq!(&base[..19], AUTHENTICATION_DOMAIN); assert_eq!(base[19], CARRIER_AUTHENTICATION_KIND); @@ -2844,9 +2918,9 @@ mod tests { assert_eq!( hex::encode(mac_statement), concat!( - "53544152464953485f5242435f4441475f56310003", + "53544152464953485f5242435f4441475f56320003", "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", - "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", + "82804ac9a25b89ad8098c52ec0ec7cfe4250d23d672a627bdb5795eda8ec4b98", "000300000002", "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f", "0002" @@ -2856,7 +2930,7 @@ mod tests { let tag = key.compute_rbc_tag(&mac_statement); assert_eq!( hex::encode(tag.as_ref()), - "118209f3c2c3025918ae7f60fe5a04a94e639cd06d910d89c483035c014fff02" + "d0beb7274a49c4d1b26f0986a370c3455e04cf899ca2e0d4d1c8ec0707746f35" ); for (field, offset) in [ ("domain", 0), diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index 344eccb1..b89b0642 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -29,11 +29,13 @@ use super::{ carrier_genesis_reference, }; -/// Executable-model runahead bound. This is deliberately a model parameter, -/// not a production protocol constant; the runtime value remains a proof and -/// benchmarking decision. +/// Executable-model runahead bounds. Admission stays close to the exact local +/// clock, while the wider authenticated-retention window lets a temporarily +/// descheduled validator catch up without forcing the healthy quorum to pace +/// itself to the slowest peer. These remain prototype resource parameters, +/// not production protocol constants. pub const EXECUTABLE_MODEL_ADMISSION_WINDOW_V1: RoundNumber = 2; -pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 4; +pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 64; const MODEL_LINEAGE_DERIVE_CONTEXT: &str = "starfish-rbc-dag-model-lineage-v1"; type ModelLineage = [u8; 32]; @@ -44,6 +46,23 @@ enum IngressAuthentication { CandidateOnly, } +/// Safety basis for the optimistic RBC fast-delivery latch. +/// +/// The first three predicates are authoritative delivery rules: the sender is +/// known honest, or the author-excluding optimistic ECHO threshold proves the +/// value unique and forces the VOTE/ACK/READY fallback to terminate on it. +/// `Delivered` records the slower `Q`-READY path when no fast predicate fired. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DeliveryPromiseBasisV1 { + LocalFixed, + /// The target author has stake greater than the maximum Byzantine stake, + /// so a receiver-authenticated author value cannot equivocate. + HonestAuthor, + /// The author-excluding optimistic ECHO threshold `O = M + b` was met. + OptimisticEcho, + Delivered, +} + /// Observable effects of one deterministic reducer transition. #[derive(Clone, Debug, Eq, PartialEq)] pub enum ModelEffect { @@ -54,6 +73,11 @@ pub enum ModelEffect { }, /// The local Bracha instance delivered this exact carrier value. Delivered(BlockReference), + /// This exact value satisfied an authoritative optimistic-delivery + /// predicate. The slower `Delivered` effect still records `Q`-READY + /// certification independently. Projection additionally requires DA and + /// an exact closed carrier prefix. + DeliveryPromised(BlockReference), /// One exact author prefix advanced by one carrier. PrefixAdvanced { authority: AuthorityIndex, @@ -72,7 +96,8 @@ pub enum ModelEffect { pub enum ModelTraceEvent { /// The first authenticated value selected for a remote carrier slot. AdmissionLocked(BlockReference), - /// A locally generated ECHO or READY became slot-global and immutable. + /// A locally generated ECHO, VOTE, ACK, or READY became slot-global and + /// immutable for its phase. LocalPhaseLocked(RbcPhaseStatementV1), /// One exact entry of an enclosing carrier's authenticated phase log is /// about to be applied. Any lock enabled by that entry follows this event. @@ -104,6 +129,13 @@ pub enum ModelTraceEvent { consensus_round: RoundNumber, choice: LeaderChoiceV1, }, + /// One safety-preserving promise predicate became true. + /// This lock is emitted at most once for an exact carrier and immediately + /// precedes its `DeliveryPromised` effect. + DeliveryPromiseLocked { + target: BlockReference, + basis: DeliveryPromiseBasisV1, + }, /// Bracha delivery became slot-global and immutable. DeliveryLocked(BlockReference), /// Existing non-durable output retained in its exact reducer order. @@ -193,6 +225,7 @@ pub struct CarrierLifecycle { pub admitted: bool, pub phase_batch_processed: bool, pub delivered: bool, + pub certified_delivered: bool, pub data_available: bool, pub prefix_closed: bool, } @@ -272,6 +305,7 @@ struct CarrierRecord { admitted: bool, phase_batch_cursor: usize, delivered: bool, + certified_delivered: bool, data_available: bool, prefix_closed: bool, } @@ -284,6 +318,7 @@ impl CarrierRecord { admitted: false, phase_batch_cursor: 0, delivered: false, + certified_delivered: false, data_available, prefix_closed: false, } @@ -296,6 +331,7 @@ impl CarrierRecord { phase_batch_processed: self.phase_batch_cursor == self.carrier.header().phase_batch().len(), delivered: self.delivered, + certified_delivered: self.certified_delivered, data_available: self.data_available, prefix_closed: self.prefix_closed, } @@ -305,25 +341,35 @@ impl CarrierRecord { #[derive(Clone, Debug, Default, Eq, PartialEq)] struct RbcCandidateState { echoes: BTreeSet, + votes: BTreeSet, + acks: BTreeSet, readies: BTreeSet, - echo_quorum_observed: bool, - ready_validity_observed: bool, - ready_quorum_observed: bool, requested_holders: BTreeSet, } impl RbcCandidateState { fn holders(&self) -> BTreeSet { - self.echoes.union(&self.readies).copied().collect() + self.echoes + .iter() + .chain(&self.votes) + .chain(&self.acks) + .chain(&self.readies) + .copied() + .collect() } } #[derive(Clone, Debug, Default, Eq, PartialEq)] struct RbcSlotState { echoed: Option, + voted: Option, + acked: Option, readied: Option, delivered: Option, + certified_delivered: Option, echo_by_sender: BTreeMap, + vote_by_sender: BTreeMap, + ack_by_sender: BTreeMap, ready_by_sender: BTreeMap, candidates: BTreeMap, } @@ -331,15 +377,37 @@ struct RbcSlotState { #[derive(Clone, Copy)] enum RbcAction { NeedCarrier, + SendVote, + SendAck, SendReady, - Deliver, + CertifyDelivery, None, } +/// Exact integer thresholds for one target-author slot. +/// +/// `fault = floor((W - 1) / 3)` is the greatest integer Byzantine stake +/// strictly below one third. ECHO, VOTE, and ACK exclude the target author. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct RbcThresholds { + fault: Stake, + ready_validity: Stake, + ready_quorum: Stake, + optimistic: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct OptimisticThresholds { + vote_from_echo: Stake, + converge: Stake, + promise_from_echo: Stake, +} + /// Pure local state machine used by the milestone-two simulations. #[derive(Clone)] pub struct RbcDagModel { committee: Arc, + total_committee_stake: Stake, committee_id: RbcDagCommitteeId, context: RbcDagContextV1, own_authority: AuthorityIndex, @@ -355,6 +423,7 @@ pub struct RbcDagModel { pending_phases: VecDeque, pending_phase_set: BTreeSet, pending_delivered_batch_replays: VecDeque, + delivery_promises: BTreeMap, prefix_tips: Vec, included_frontier: Vec>, included: BTreeSet, @@ -374,6 +443,16 @@ impl RbcDagModel { if context.committee_id() != committee_id { return Err(ModelError::ContextMismatch); } + let total_committee_stake = + committee.authorities().try_fold(0u64, |total, authority| { + total + .checked_add( + committee + .get_stake(authority) + .ok_or(ModelError::UnknownAuthority(authority))?, + ) + .ok_or(ModelError::InvalidCommittee) + })?; let prefix_tips = committee .authorities() .map(carrier_genesis_reference) @@ -381,6 +460,7 @@ impl RbcDagModel { Ok(Self { included_frontier: vec![None; committee.len()], committee, + total_committee_stake, committee_id, context, own_authority, @@ -396,6 +476,7 @@ impl RbcDagModel { pending_phases: VecDeque::new(), pending_phase_set: BTreeSet::new(), pending_delivered_batch_replays: VecDeque::new(), + delivery_promises: BTreeMap::new(), prefix_tips, included: BTreeSet::new(), }) @@ -570,7 +651,7 @@ impl RbcDagModel { let limit = self .committee .len() - .saturating_mul(4) + .saturating_mul(6) .min(MAX_PHASE_STATEMENTS_V1); self.pending_phases .iter() @@ -634,7 +715,7 @@ impl RbcDagModel { /// Fix a locally authored carrier. Its phase batch must be the exact /// bounded FIFO prefix returned by [`Self::pending_phase_batch`]; newly - /// generated ECHO is left for a later carrier. + /// generated phase statements are left for a later carrier. pub fn start_local_carrier( &mut self, authenticated: LocallyAuthenticatedCarrierV1, @@ -698,8 +779,8 @@ impl RbcDagModel { .collect(); let reference = carrier.reference(); // Preflight all fallible ingress checks before the proof-critical - // write order. The exact local carrier must be fixed before its local - // ECHO is authorized or any embedded phase statement is exposed. + // write order. The exact local carrier must be fixed before any local + // phase statement is authorized or any embedded statement is exposed. self.preflight_receive(&carrier)?; self.own_fixed.insert(round, reference); log.proof(ModelTraceEvent::LocalCarrierFixed(reference)); @@ -713,6 +794,7 @@ impl RbcDagModel { choice: vertex.leader_choice(), }); } + self.lock_delivery_promise(reference, DeliveryPromiseBasisV1::LocalFixed, log); for statement in &expected_phase_batch { self.pending_phase_set.remove(statement); } @@ -831,10 +913,21 @@ impl RbcDagModel { true } }; + if selected && self.target_author_is_honest(reference.authority) { + // This capability authenticates the exact target-author bytes + // to this receiver. If the author's stake exceeds F, the + // fault model itself rules out author equivocation. + self.lock_delivery_promise(reference, DeliveryPromiseBasisV1::HonestAuthor, log); + } if selected && self.in_admission_window(reference.round) { self.promote_authenticated(reference, log); } } + // A locally fixed promise is persisted before the carrier is inserted + // above. Activate every already-locked fast-delivery predicate only + // after exact content exists, and defer phase-batch replay until the + // current reducer input finishes. + self.activate_delivery_promise(reference, log); self.maybe_advance_fast_clock(log); self.drain_delivered_phase_batches(log); } @@ -904,6 +997,33 @@ impl RbcDagModel { .and_then(|slot| slot.delivered) } + /// Return the exact value that reached the fallback `Q`-READY + /// certificate, independently of an earlier optimistic delivery. + pub fn certified_delivered( + &self, + authority: AuthorityIndex, + round: RoundNumber, + ) -> Option { + self.rbc_slots + .get(&(round, authority)) + .and_then(|slot| slot.certified_delivered) + } + + pub fn certified_delivery_count(&self) -> usize { + self.rbc_slots + .values() + .filter(|slot| slot.certified_delivered.is_some()) + .count() + } + + /// Return the first durable fast-delivery basis for this exact carrier. + pub fn delivery_promise_basis( + &self, + reference: &BlockReference, + ) -> Option { + self.delivery_promises.get(reference).copied() + } + pub fn prefix_tip(&self, authority: AuthorityIndex) -> Option { let tip = *self.prefix_tips.get(authority as usize)?; (tip.round > 0).then_some(tip) @@ -1074,6 +1194,56 @@ impl RbcDagModel { }) } + fn voters_stake_excluding( + &self, + voters: &BTreeSet, + excluded: AuthorityIndex, + ) -> Stake { + voters.iter().fold(0, |stake, authority| { + if *authority == excluded { + stake + } else { + stake.saturating_add(self.authority_stake(*authority)) + } + }) + } + + fn rbc_thresholds(&self, target_author: AuthorityIndex) -> Option { + let author_stake = self.committee.get_stake(target_author)?; + let fault = self.total_committee_stake.checked_sub(1)? / 3; + let ready_validity = fault.checked_add(1)?; + let ready_quorum = self.total_committee_stake.checked_sub(fault)?; + let optimistic = if author_stake <= fault { + let non_author_stake = self.total_committee_stake.checked_sub(author_stake)?; + let residual_fault = fault.checked_sub(author_stake)?; + let vote_from_echo = non_author_stake.checked_div(2)?.checked_add(1)?; + // floor((U + b) / 2) without overflowing the intermediate sum. + let converge = (non_author_stake / 2) + .checked_add(residual_fault / 2)? + .checked_add((non_author_stake % 2 + residual_fault % 2) / 2)? + .checked_add(1)?; + let promise_from_echo = vote_from_echo.checked_add(residual_fault)?; + Some(OptimisticThresholds { + vote_from_echo, + converge, + promise_from_echo, + }) + } else { + None + }; + Some(RbcThresholds { + fault, + ready_validity, + ready_quorum, + optimistic, + }) + } + + fn target_author_is_honest(&self, target_author: AuthorityIndex) -> bool { + self.rbc_thresholds(target_author) + .is_some_and(|thresholds| thresholds.optimistic.is_none()) + } + fn rbc_slot_mut(&mut self, reference: BlockReference) -> &mut RbcSlotState { self.rbc_slots .entry((reference.round, reference.authority)) @@ -1082,6 +1252,15 @@ impl RbcDagModel { fn authorize_local_echo(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; + if own == reference.authority { + // The target author is excluded from ECHO/VOTE/ACK. A locally + // fixed high-stake author instead seeds READY: its stake is at + // least F+1, so every correct receiver can safely amplify it. + if self.target_author_is_honest(reference.authority) { + self.authorize_local_ready(reference, log); + } + return; + } let slot = self.rbc_slot_mut(reference); if slot.echoed.is_some() { return; @@ -1099,6 +1278,25 @@ impl RbcDagModel { self.drive_rbc(reference, log); } + fn authorize_local_ready(&mut self, reference: BlockReference, log: &mut TransitionLog) { + let own = self.own_authority; + let slot = self.rbc_slot_mut(reference); + if slot.readied.is_some() { + return; + } + slot.readied = Some(reference); + slot.ready_by_sender.insert(own, reference); + slot.candidates + .entry(reference) + .or_default() + .readies + .insert(own); + let statement = RbcPhaseStatementV1::Ready { target: reference }; + log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); + self.queue_local_phase(statement); + self.drive_rbc(reference, log); + } + fn queue_local_phase(&mut self, statement: RbcPhaseStatementV1) { if self.pending_phase_set.insert(statement) { self.pending_phases.push_back(statement); @@ -1176,12 +1374,29 @@ impl RbcDagModel { return; } let target = statement.target(); + if !self.committee.known_authority(target.authority) || target.round == 0 { + return; + } + if sender == target.authority + && matches!( + statement, + RbcPhaseStatementV1::Echo { .. } + | RbcPhaseStatementV1::Vote { .. } + | RbcPhaseStatementV1::Ack { .. } + ) + { + // The broadcaster may equivocate. Its stake is deliberately + // excluded from every optimistic certificate phase. + return; + } if sender == self.own_authority { let authorized = self .rbc_slots .get(&(target.round, target.authority)) .is_some_and(|slot| match statement { RbcPhaseStatementV1::Echo { .. } => slot.echoed == Some(target), + RbcPhaseStatementV1::Vote { .. } => slot.voted == Some(target), + RbcPhaseStatementV1::Ack { .. } => slot.acked == Some(target), RbcPhaseStatementV1::Ready { .. } => slot.readied == Some(target), }); if !authorized { @@ -1194,6 +1409,8 @@ impl RbcDagModel { let slot = self.rbc_slot_mut(target); let senders = match statement { RbcPhaseStatementV1::Echo { .. } => &mut slot.echo_by_sender, + RbcPhaseStatementV1::Vote { .. } => &mut slot.vote_by_sender, + RbcPhaseStatementV1::Ack { .. } => &mut slot.ack_by_sender, RbcPhaseStatementV1::Ready { .. } => &mut slot.ready_by_sender, }; match senders.get(&sender) { @@ -1208,6 +1425,12 @@ impl RbcDagModel { RbcPhaseStatementV1::Echo { .. } => { candidate.echoes.insert(sender); } + RbcPhaseStatementV1::Vote { .. } => { + candidate.votes.insert(sender); + } + RbcPhaseStatementV1::Ack { .. } => { + candidate.acks.insert(sender); + } RbcPhaseStatementV1::Ready { .. } => { candidate.readies.insert(sender); } @@ -1215,6 +1438,53 @@ impl RbcDagModel { self.drive_rbc(target, log); } + fn lock_delivery_promise( + &mut self, + target: BlockReference, + basis: DeliveryPromiseBasisV1, + log: &mut TransitionLog, + ) { + if self.delivery_promises.contains_key(&target) { + return; + } + if self + .rbc_slots + .get(&(target.round, target.authority)) + .and_then(|slot| slot.delivered) + .is_some_and(|delivered| delivered != target) + { + // Integrity is enforced locally even though the optimistic and + // fallback quorum proofs already rule out conflicting honest + // deliveries. + return; + } + self.delivery_promises.insert(target, basis); + log.proof(ModelTraceEvent::DeliveryPromiseLocked { target, basis }); + log.effect(ModelEffect::DeliveryPromised(target)); + self.activate_delivery_promise(target, log); + } + + fn activate_delivery_promise(&mut self, target: BlockReference, log: &mut TransitionLog) { + let Some(basis) = self.delivery_promises.get(&target).copied() else { + return; + }; + if basis == DeliveryPromiseBasisV1::Delivered || !self.carriers.contains_key(&target) { + return; + } + let slot = self.rbc_slot_mut(target); + match slot.delivered { + Some(existing) if existing != target => return, + Some(_) => return, + None => slot.delivered = Some(target), + } + self.carriers + .get_mut(&target) + .expect("fast delivery requires exact canonical carrier content") + .delivered = true; + self.pending_delivered_batch_replays.push_back(target); + self.drive_prefix(target.authority, log); + } + fn drive_rbc(&mut self, target: BlockReference, log: &mut TransitionLog) { let slot_key = (target.round, target.authority); if !self @@ -1224,34 +1494,57 @@ impl RbcDagModel { { // Merely staging canonical content is not authenticated RBC // evidence. Candidate state is allocated only by a locally - // authorized ECHO or an embedded ECHO/READY statement. + // authorized phase or an embedded phase statement. return; } + let Some(thresholds) = self.rbc_thresholds(target.authority) else { + return; + }; loop { let header_available = self.carriers.contains_key(&target); - let q = self.committee.quorum_threshold(); - let v = self.committee.validity_threshold(); + let (echo_stake, vote_stake, ack_stake, ready_stake) = { + let candidate = self + .rbc_slots + .get(&slot_key) + .and_then(|slot| slot.candidates.get(&target)) + .expect("the candidate remains allocated"); + ( + self.voters_stake_excluding(&candidate.echoes, target.authority), + self.voters_stake_excluding(&candidate.votes, target.authority), + self.voters_stake_excluding(&candidate.acks, target.authority), + self.voters_stake(&candidate.readies), + ) + }; + let (vote_trigger, ack_trigger, optimistic_ready_trigger, promise_trigger) = thresholds + .optimistic + .map_or((false, false, false, false), |optimistic| { + ( + echo_stake >= optimistic.vote_from_echo, + echo_stake >= optimistic.converge || vote_stake >= optimistic.converge, + ack_stake >= optimistic.converge, + echo_stake >= optimistic.promise_from_echo, + ) + }); + let ready_trigger = + optimistic_ready_trigger || ready_stake >= thresholds.ready_validity; + let deliver_trigger = ready_stake >= thresholds.ready_quorum; + let promise_missing = !self.delivery_promises.contains_key(&target); + + // A promise names exact canonical content, never a digest learned + // only from phase evidence. + if header_available && promise_missing && promise_trigger { + self.lock_delivery_promise(target, DeliveryPromiseBasisV1::OptimisticEcho, log); + } + let can_send_optimistic_phase = self.own_authority != target.authority; let action = { - let echo_stake; - let ready_stake; - { - let slot = self.rbc_slot_mut(target); - let candidate = slot.candidates.entry(target).or_default(); - echo_stake = candidate.echoes.clone(); - ready_stake = candidate.readies.clone(); - } - let echo_stake = self.voters_stake(&echo_stake); - let ready_stake = self.voters_stake(&ready_stake); let slot = self.rbc_slot_mut(target); let candidate = slot.candidates.entry(target).or_default(); - candidate.echo_quorum_observed |= echo_stake >= q; - candidate.ready_validity_observed |= ready_stake >= v; - candidate.ready_quorum_observed |= ready_stake >= q; - let ready_trigger = - candidate.echo_quorum_observed || candidate.ready_validity_observed; let needs_header = !header_available - && ((slot.readied.is_none() && ready_trigger) - || (slot.delivered.is_none() && candidate.ready_quorum_observed)); + && ((slot.voted.is_none() && can_send_optimistic_phase && vote_trigger) + || (slot.acked.is_none() && can_send_optimistic_phase && ack_trigger) + || (slot.readied.is_none() && ready_trigger) + || (slot.certified_delivered.is_none() && deliver_trigger) + || (promise_missing && promise_trigger)); if needs_header { let holders = candidate.holders(); if holders != candidate.requested_holders { @@ -1260,13 +1553,23 @@ impl RbcDagModel { } else { RbcAction::None } + } else if header_available + && slot.voted.is_none() + && can_send_optimistic_phase + && vote_trigger + { + RbcAction::SendVote + } else if header_available + && slot.acked.is_none() + && can_send_optimistic_phase + && ack_trigger + { + RbcAction::SendAck } else if header_available && slot.readied.is_none() && ready_trigger { RbcAction::SendReady - } else if header_available - && slot.delivered.is_none() - && candidate.ready_quorum_observed + } else if header_available && slot.certified_delivered.is_none() && deliver_trigger { - RbcAction::Deliver + RbcAction::CertifyDelivery } else { RbcAction::None } @@ -1285,28 +1588,44 @@ impl RbcDagModel { log.effect(ModelEffect::NeedCarrier { target, holders }); break; } - RbcAction::SendReady => { + RbcAction::SendVote => { let own = self.own_authority; let slot = self.rbc_slot_mut(target); - slot.readied = Some(target); - slot.ready_by_sender.insert(own, target); - slot.candidates - .entry(target) - .or_default() - .readies - .insert(own); - let statement = RbcPhaseStatementV1::Ready { target }; + slot.voted = Some(target); + slot.vote_by_sender.insert(own, target); + slot.candidates.entry(target).or_default().votes.insert(own); + let statement = RbcPhaseStatementV1::Vote { target }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); } - RbcAction::Deliver => { - self.rbc_slot_mut(target).delivered = Some(target); + RbcAction::SendAck => { + let own = self.own_authority; + let slot = self.rbc_slot_mut(target); + slot.acked = Some(target); + slot.ack_by_sender.insert(own, target); + slot.candidates.entry(target).or_default().acks.insert(own); + let statement = RbcPhaseStatementV1::Ack { target }; + log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); + self.queue_local_phase(statement); + } + RbcAction::SendReady => { + self.authorize_local_ready(target, log); + } + RbcAction::CertifyDelivery => { + let slot = self.rbc_slot_mut(target); + if slot.delivered.is_some_and(|delivered| delivered != target) { + break; + } + slot.delivered = Some(target); + slot.certified_delivered = Some(target); let record = self .carriers .get_mut(&target) .expect("delivery requires exact canonical carrier content"); record.delivered = true; + record.certified_delivered = true; log.proof(ModelTraceEvent::DeliveryLocked(target)); + self.lock_delivery_promise(target, DeliveryPromiseBasisV1::Delivered, log); log.effect(ModelEffect::Delivered(target)); self.pending_delivered_batch_replays.push_back(target); self.drive_prefix(target.authority, log); @@ -1590,7 +1909,9 @@ mod tests { .map(|authority| model(Arc::clone(&committee), authority)) .collect(); let mut rounds = Vec::new(); - for round in 1..=6 { + // ECHO, VOTE/ACK, and READY are embedded in later carriers. Seven + // carrier rounds make the first four rounds mature end-to-end. + for round in 1..=7 { rounds.push(run_honest_round(&mut models, round)); } for model in &models { @@ -1939,6 +2260,7 @@ mod tests { assert_eq!(replayed.authenticated_by_slot, live.authenticated_by_slot); assert_eq!(replayed.admitted_by_slot, live.admitted_by_slot); assert_eq!(replayed.rbc_slots, live.rbc_slots); + assert_eq!(replayed.delivery_promises, live.delivery_promises); assert_eq!(replayed.pending_phases, live.pending_phases); assert_eq!(replayed.pending_phase_set, live.pending_phase_set); for reference in references { @@ -1962,10 +2284,12 @@ mod tests { let mut model = model(committee, 0); model.local_carrier_round = 4; let mut queued = Vec::new(); - for round in 1..=3 { + for round in 1..=2 { for author in 0..4 { let target = BlockReference::new_test(author, round); queued.push(RbcPhaseStatementV1::Echo { target }); + queued.push(RbcPhaseStatementV1::Vote { target }); + queued.push(RbcPhaseStatementV1::Ack { target }); queued.push(RbcPhaseStatementV1::Ready { target }); } } @@ -1973,8 +2297,8 @@ mod tests { model.queue_local_phase(*statement); } - assert_eq!(model.pending_phase_backlog_len(), 24); - assert_eq!(model.pending_phase_batch(), queued[..16]); + assert_eq!(model.pending_phase_backlog_len(), 32); + assert_eq!(model.pending_phase_batch(), queued[..24]); } #[test] @@ -2050,6 +2374,545 @@ mod tests { ); } + #[test] + fn local_fix_fast_delivers_exact_content_once_without_q_ready_certificate() { + let committee = committee(4); + let model = model(Arc::clone(&committee), 3); + let (own_prev, weak_parents) = model.local_parent_set().unwrap(); + let carrier = candidate( + &committee, + 3, + 1, + own_prev, + weak_parents, + model.pending_phase_batch(), + 0xF0, + ) + .unwrap(); + let target = carrier.reference(); + let plan = model + .plan_input(ModelInputRecord::LocalCarrierFixed(authenticate_local( + &committee, &carrier, + ))) + .unwrap(); + + let fixed = plan + .trace() + .iter() + .position(|event| *event == ModelTraceEvent::LocalCarrierFixed(target)) + .unwrap(); + let promised = plan + .trace() + .iter() + .position(|event| { + *event + == ModelTraceEvent::DeliveryPromiseLocked { + target, + basis: DeliveryPromiseBasisV1::LocalFixed, + } + }) + .unwrap(); + assert!(fixed < promised); + assert_eq!( + plan.effects() + .iter() + .filter(|effect| **effect == ModelEffect::DeliveryPromised(target)) + .count(), + 1 + ); + + let mut committed = model; + committed.commit_plan(plan).unwrap(); + assert_eq!( + committed.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::LocalFixed) + ); + assert_eq!(committed.delivered(3, 1), Some(target)); + assert_eq!(committed.certified_delivered(3, 1), None); + assert!(committed.lifecycle(&target).unwrap().delivered); + assert!(!committed.lifecycle(&target).unwrap().certified_delivered); + assert!(!committed.lifecycle(&target).unwrap().prefix_closed); + } + + #[test] + fn weighted_thresholds_follow_the_target_author_formula() { + let cases = [ + (vec![1, 1, 1, 1], 0, (1, 2, 3, Some((2, 2, 2)))), + (vec![1, 1, 1, 1, 1, 1, 1], 0, (2, 3, 5, Some((4, 4, 5)))), + (vec![2, 1, 1, 1, 1, 1], 0, (2, 3, 5, Some((3, 3, 3)))), + (vec![1, 2, 2, 2], 0, (2, 3, 5, Some((4, 4, 5)))), + (vec![3, 1, 1, 1], 0, (1, 2, 5, None)), + ]; + + for (stakes, target_author, (fault, validity, quorum, optimistic)) in cases { + let committee = Committee::new_test(stakes); + let model = model(committee, target_author); + let thresholds = model.rbc_thresholds(target_author).unwrap(); + assert_eq!(thresholds.fault, fault); + assert_eq!(thresholds.ready_validity, validity); + assert_eq!(thresholds.ready_quorum, quorum); + assert_eq!( + thresholds.optimistic.map(|thresholds| ( + thresholds.vote_from_echo, + thresholds.converge, + thresholds.promise_from_echo, + )), + optimistic + ); + } + } + + #[test] + fn exhaustive_weighted_echo_subsets_promise_exactly_at_o() { + for stakes in [ + vec![1, 1, 1, 1, 1, 1, 1], + vec![2, 1, 1, 1, 1, 1], + vec![1, 2, 1, 2, 1], + vec![2, 3, 1, 1, 1, 1, 1], + ] { + let committee = Committee::new_test(stakes); + let template = model(Arc::clone(&committee), 0); + let threshold = template + .rbc_thresholds(0) + .unwrap() + .optimistic + .unwrap() + .promise_from_echo; + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A0).unwrap(); + let target = carrier.reference(); + let non_authors: Vec<_> = committee + .authorities() + .filter(|sender| *sender != 0) + .collect(); + + for mask in 0usize..(1usize << non_authors.len()) { + for reverse in [false, true] { + let mut model = model(Arc::clone(&committee), 0); + model.stage_candidate(carrier.clone()).unwrap(); + // The author is never counted, even if it embeds an ECHO. + record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }); + let mut selected: Vec<_> = non_authors + .iter() + .enumerate() + .filter_map(|(index, sender)| ((mask >> index) & 1 == 1).then_some(*sender)) + .collect(); + if reverse { + selected.reverse(); + } + let mut observed_stake = 0; + for sender in selected { + observed_stake += committee.get_stake(sender).unwrap(); + record_phase(&mut model, sender, RbcPhaseStatementV1::Echo { target }); + assert_eq!( + model.delivery_promise_basis(&target).is_some(), + observed_stake >= threshold + ); + } + assert_eq!( + model.delivery_promise_basis(&target), + (observed_stake >= threshold) + .then_some(DeliveryPromiseBasisV1::OptimisticEcho) + ); + } + } + } + } + + #[test] + fn exhaustive_weighted_optimistic_certificates_intersect_honestly() { + for stakes in [ + vec![1, 1, 1, 1, 1, 1, 1], + vec![2, 1, 1, 1, 1, 1], + vec![1, 2, 1, 2, 1], + vec![2, 3, 1, 1, 1, 1, 1], + ] { + let committee = Committee::new_test(stakes); + let model = model(Arc::clone(&committee), 0); + let thresholds = model.rbc_thresholds(0).unwrap(); + let optimistic = thresholds.optimistic.unwrap(); + let author_stake = committee.get_stake(0).unwrap(); + let residual_fault = thresholds.fault - author_stake; + let non_authors: Vec<_> = committee + .authorities() + .filter(|sender| *sender != 0) + .collect(); + let subset_stake = |mask: usize| { + non_authors + .iter() + .enumerate() + .filter(|(index, _)| (mask >> index) & 1 == 1) + .map(|(_, sender)| committee.get_stake(*sender).unwrap()) + .sum::() + }; + let limit = 1usize << non_authors.len(); + let certificates: Vec<_> = (0..limit) + .filter(|mask| subset_stake(*mask) >= optimistic.promise_from_echo) + .collect(); + let byzantine_sets: Vec<_> = (0..limit) + .filter(|mask| subset_stake(*mask) <= residual_fault) + .collect(); + + for first in &certificates { + for second in &certificates { + for byzantine in &byzantine_sets { + // Every pair of selective O certificates shares a + // non-author sender outside the remaining Byzantine + // budget. That honest ECHO lock forbids two values. + assert_ne!(first & second & !byzantine, 0); + } + } + } + } + } + + #[test] + fn weighted_four_phase_rules_and_q_ready_delivery_are_exact() { + let committee = committee(7); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A1).unwrap(); + let target = carrier.reference(); + + let mut from_echo = model(Arc::clone(&committee), 6); + from_echo.stage_candidate(carrier.clone()).unwrap(); + for sender in 1..=3 { + record_phase(&mut from_echo, sender, RbcPhaseStatementV1::Echo { target }); + } + assert!(from_echo.rbc_slots[&(1, 0)].voted.is_none()); + assert!(from_echo.rbc_slots[&(1, 0)].acked.is_none()); + record_phase(&mut from_echo, 4, RbcPhaseStatementV1::Echo { target }); + assert_eq!(from_echo.rbc_slots[&(1, 0)].voted, Some(target)); + assert_eq!(from_echo.rbc_slots[&(1, 0)].acked, Some(target)); + assert_eq!(from_echo.delivery_promise_basis(&target), None); + record_phase(&mut from_echo, 5, RbcPhaseStatementV1::Echo { target }); + assert_eq!( + from_echo.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::OptimisticEcho) + ); + + let mut from_vote = model(Arc::clone(&committee), 6); + from_vote.stage_candidate(carrier.clone()).unwrap(); + for sender in 1..=3 { + record_phase(&mut from_vote, sender, RbcPhaseStatementV1::Vote { target }); + } + assert!(from_vote.rbc_slots[&(1, 0)].acked.is_none()); + record_phase(&mut from_vote, 4, RbcPhaseStatementV1::Vote { target }); + assert_eq!(from_vote.rbc_slots[&(1, 0)].acked, Some(target)); + + let mut from_ack = model(Arc::clone(&committee), 6); + from_ack.stage_candidate(carrier.clone()).unwrap(); + for sender in 1..=3 { + record_phase(&mut from_ack, sender, RbcPhaseStatementV1::Ack { target }); + } + assert!(from_ack.rbc_slots[&(1, 0)].readied.is_none()); + record_phase(&mut from_ack, 4, RbcPhaseStatementV1::Ack { target }); + assert_eq!(from_ack.rbc_slots[&(1, 0)].readied, Some(target)); + + let mut from_ready = model(Arc::clone(&committee), 0); + from_ready.stage_candidate(carrier).unwrap(); + for sender in 1..=2 { + record_phase( + &mut from_ready, + sender, + RbcPhaseStatementV1::Ready { target }, + ); + } + assert!(from_ready.rbc_slots[&(1, 0)].readied.is_none()); + assert_eq!(from_ready.delivered(0, 1), None); + record_phase(&mut from_ready, 3, RbcPhaseStatementV1::Ready { target }); + assert_eq!(from_ready.rbc_slots[&(1, 0)].readied, Some(target)); + // Three remote READYs plus the local READY have stake four, below Q=5. + assert_eq!(from_ready.delivered(0, 1), None); + record_phase(&mut from_ready, 4, RbcPhaseStatementV1::Ready { target }); + assert_eq!(from_ready.delivered(0, 1), Some(target)); + } + + #[test] + fn missing_content_blocks_every_phase_and_requests_all_phase_holders() { + let committee = committee(7); + let mut model = model(Arc::clone(&committee), 6); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A2).unwrap(); + let target = carrier.reference(); + + for sender in 1..=4 { + record_phase(&mut model, sender, RbcPhaseStatementV1::Echo { target }); + } + record_phase(&mut model, 5, RbcPhaseStatementV1::Vote { target }); + record_phase(&mut model, 5, RbcPhaseStatementV1::Ack { target }); + let effects = record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target }); + assert!(matches!( + effects.as_slice(), + [ModelEffect::NeedCarrier { target: requested, holders }] + if *requested == target && holders == &[0, 1, 2, 3, 4, 5] + )); + let slot = &model.rbc_slots[&(1, 0)]; + assert!(slot.voted.is_none()); + assert!(slot.acked.is_none()); + assert!(slot.readied.is_none()); + assert!(slot.delivered.is_none()); + assert_eq!(model.delivery_promise_basis(&target), None); + + model.recover_carrier(carrier).unwrap(); + let slot = &model.rbc_slots[&(1, 0)]; + assert_eq!(slot.voted, Some(target)); + assert_eq!(slot.acked, Some(target)); + } + + #[test] + fn high_stake_honest_author_promises_on_auth_and_seeds_ready_locally() { + let committee = Committee::new_test(vec![3, 1, 1, 1]); + let mut author = model(Arc::clone(&committee), 0); + let (own_prev, weak) = author.local_parent_set().unwrap(); + let carrier = candidate( + &committee, + 0, + 1, + own_prev, + weak, + author.pending_phase_batch(), + 0x1A3, + ) + .unwrap(); + let target = carrier.reference(); + let effects = author + .start_local_carrier(authenticate_local(&committee, &carrier)) + .unwrap(); + assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); + assert_eq!( + author.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::LocalFixed) + ); + assert_eq!(author.rbc_slots[&(1, 0)].readied, Some(target)); + assert!( + author + .pending_phases + .contains(&RbcPhaseStatementV1::Ready { target }) + ); + + let mut receiver = model(Arc::clone(&committee), 3); + receiver.stage_candidate(carrier.clone()).unwrap(); + assert_eq!(receiver.delivery_promise_basis(&target), None); + let effects = receiver + .receive_authenticated(authenticate_for(&committee, &carrier, 3)) + .unwrap(); + assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); + assert_eq!( + receiver.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::HonestAuthor) + ); + assert!(receiver.rbc_slots[&(1, 0)].readied.is_none()); + assert_eq!(receiver.delivered(0, 1), Some(target)); + assert_eq!(receiver.certified_delivered(0, 1), None); + + record_phase(&mut receiver, 0, RbcPhaseStatementV1::Ready { target }); + assert_eq!(receiver.rbc_slots[&(1, 0)].readied, Some(target)); + assert_eq!(receiver.delivered(0, 1), Some(target)); + assert_eq!(receiver.certified_delivered(0, 1), None); + record_phase(&mut receiver, 1, RbcPhaseStatementV1::Ready { target }); + assert_eq!(receiver.delivered(0, 1), Some(target)); + assert_eq!(receiver.certified_delivered(0, 1), Some(target)); + + // Even a test-only forged second author capability cannot bypass the + // receiver's exact authenticated slot lock. + let (own_prev, weak) = genesis_parents(&committee, 0); + let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A4).unwrap(); + let conflicting_ref = conflicting.reference(); + receiver + .receive_authenticated(authenticate_for(&committee, &conflicting, 3)) + .unwrap(); + assert_eq!(receiver.delivery_promise_basis(&conflicting_ref), None); + } + + #[test] + fn raw_q_that_counts_the_target_author_is_not_an_optimistic_promise() { + let committee = committee(7); + let mut model = model(Arc::clone(&committee), 0); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF1).unwrap(); + let target = carrier.reference(); + model.stage_candidate(carrier).unwrap(); + + // W=7, F=2, a=1 gives O=5. A raw Q=5 that includes the + // equivocating target author contains only four admissible ECHOs. + assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }).is_empty()); + for sender in 1..=4 { + assert!( + record_phase(&mut model, sender, RbcPhaseStatementV1::Echo { target }).is_empty() + ); + } + assert_eq!(model.delivery_promise_basis(&target), None); + assert_eq!(model.delivered(0, 1), None); + + let effects = record_phase(&mut model, 5, RbcPhaseStatementV1::Echo { target }); + assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); + assert_eq!( + model.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::OptimisticEcho) + ); + assert_eq!(model.delivered(0, 1), Some(target)); + assert_eq!(model.certified_delivered(0, 1), None); + + // Exact replay and a conflicting later ECHO are both idempotent and + // cannot emit a second promise. + assert!(record_phase(&mut model, 5, RbcPhaseStatementV1::Echo { target }).is_empty()); + let mut conflicting = target; + conflicting.digest = crate::types::BlockDigest::from([0xF2; 32]); + assert!( + record_phase( + &mut model, + 5, + RbcPhaseStatementV1::Echo { + target: conflicting, + }, + ) + .is_empty() + ); + assert_eq!(model.delivery_promises.len(), 1); + } + + #[test] + fn selective_and_equivocating_echoes_cannot_promise_two_values() { + let committee = committee(7); + let mut model = model(Arc::clone(&committee), 0); + let (own_prev, weak) = genesis_parents(&committee, 0); + let first = candidate(&committee, 0, 1, own_prev, weak.clone(), Vec::new(), 0xF3).unwrap(); + let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF4).unwrap(); + let first_ref = first.reference(); + let conflicting_ref = conflicting.reference(); + + model.stage_candidate(first).unwrap(); + model.stage_candidate(conflicting).unwrap(); + for sender in 1..=4 { + record_phase( + &mut model, + sender, + RbcPhaseStatementV1::Echo { target: first_ref }, + ); + } + for sender in 5..=6 { + record_phase( + &mut model, + sender, + RbcPhaseStatementV1::Echo { + target: conflicting_ref, + }, + ); + } + // Sender 1 equivocates after locking the first value. The second + // statement is ignored slot-globally for ECHO. + record_phase( + &mut model, + 1, + RbcPhaseStatementV1::Echo { + target: conflicting_ref, + }, + ); + + assert_eq!( + model.rbc_slots.get(&(1, 0)).unwrap().candidates[&first_ref] + .echoes + .len(), + 4 + ); + assert_eq!( + model.rbc_slots[&(1, 0)].candidates[&conflicting_ref] + .echoes + .len(), + 2 + ); + assert_eq!(model.delivery_promise_basis(&first_ref), None); + assert_eq!(model.delivery_promise_basis(&conflicting_ref), None); + assert!(model.delivery_promises.is_empty()); + } + + #[test] + fn delivered_fallback_promises_before_the_delivered_effect() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 3); + let (own_prev, weak) = genesis_parents(&committee, 0); + let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF5).unwrap(); + let target = carrier.reference(); + model.stage_candidate(carrier).unwrap(); + + assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target }).is_empty()); + let effects = record_phase(&mut model, 1, RbcPhaseStatementV1::Ready { target }); + assert_eq!( + effects, + vec![ + ModelEffect::DeliveryPromised(target), + ModelEffect::Delivered(target), + ] + ); + assert_eq!( + model.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::Delivered) + ); + } + + #[test] + fn promised_effects_and_locks_replay_deterministically_from_typed_inputs() { + let committee = committee(4); + let context = context(&committee); + let mut live = RbcDagModel::new(Arc::clone(&committee), 3, context).unwrap(); + let (own_prev, weak) = genesis_parents(&committee, 0); + let target_carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF6).unwrap(); + let target = target_carrier.reference(); + let mut records = vec![ModelInputRecord::AuthenticatedIngress(authenticate_for( + &committee, + &target_carrier, + 3, + ))]; + + for author in 0..3 { + let outer = candidate( + &committee, + author, + 2, + BlockReference::new_test(author, 1), + committee + .authorities() + .filter(|other| *other != author) + .take(2) + .map(|other| BlockReference::new_test(other, 1)) + .collect(), + vec![RbcPhaseStatementV1::Echo { target }], + 0xF7 + u64::from(author), + ) + .unwrap(); + records.push(ModelInputRecord::AuthenticatedIngress(authenticate_for( + &committee, &outer, 3, + ))); + } + + let mut live_trace = Vec::new(); + for record in records.iter().cloned() { + let plan = live.plan_input(record).unwrap(); + live_trace.extend_from_slice(plan.trace()); + live.commit_plan(plan).unwrap(); + } + let (replayed, replay_trace) = + RbcDagModel::replay_from_records(committee, 3, context, records).unwrap(); + + assert_eq!(replay_trace, live_trace); + assert_eq!(replayed.delivery_promises, live.delivery_promises); + assert_eq!( + replay_trace + .iter() + .filter(|event| { + **event == ModelTraceEvent::Effect(ModelEffect::DeliveryPromised(target)) + }) + .count(), + 1 + ); + assert_eq!( + replayed.delivery_promise_basis(&target), + Some(DeliveryPromiseBasisV1::OptimisticEcho) + ); + assert_eq!(replayed.delivered(0, 1), Some(target)); + assert_eq!(replayed.certified_delivered(0, 1), None); + } + #[test] fn threshold_before_header_requests_then_recovers_exact_carrier() { let committee = committee(4); @@ -2058,7 +2921,6 @@ mod tests { let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 1).unwrap(); let target = carrier.reference(); - assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }).is_empty()); assert!(record_phase(&mut model, 1, RbcPhaseStatementV1::Echo { target }).is_empty()); assert!(matches!( record_phase( @@ -2068,14 +2930,20 @@ mod tests { ) .as_slice(), [ModelEffect::NeedCarrier { target: requested, holders }] - if *requested == target && holders == &[0, 1, 2] + if *requested == target && holders == &[1, 2] )); - model.recover_carrier(carrier).unwrap(); + let effects = model.recover_carrier(carrier).unwrap(); + assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); assert!( model .pending_phases - .contains(&RbcPhaseStatementV1::Ready { target }) + .contains(&RbcPhaseStatementV1::Vote { target }) + ); + assert!( + model + .pending_phases + .contains(&RbcPhaseStatementV1::Ack { target }) ); let lifecycle = model.lifecycle(&target).unwrap(); assert!(!lifecycle.authenticated); @@ -2220,6 +3088,8 @@ mod tests { let target = BlockReference::new_test(0, 1); assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Echo { target }).is_empty()); + assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Vote { target }).is_empty()); + assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Ack { target }).is_empty()); assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Ready { target }).is_empty()); assert!(model.rbc_slots.is_empty()); } @@ -2261,8 +3131,8 @@ mod tests { [ModelEffect::NeedCarrier { target, .. }] if *target == first )); let slot = model.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.echo_by_sender.len(), 3); - assert_eq!(slot.echo_by_sender[&0], first); + assert_eq!(slot.echo_by_sender.len(), 2); + assert_eq!(slot.echo_by_sender[&1], first); assert!(!slot.candidates.contains_key(&conflicting)); record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target: first }); @@ -2276,6 +3146,26 @@ mod tests { let slot = model.rbc_slots.get(&(1, 0)).unwrap(); assert_eq!(slot.ready_by_sender[&0], first); assert!(!slot.candidates.contains_key(&conflicting)); + + for statement in [ + RbcPhaseStatementV1::Vote { target: first }, + RbcPhaseStatementV1::Ack { target: first }, + ] { + record_phase(&mut model, 1, statement); + let conflicting_statement = match statement { + RbcPhaseStatementV1::Vote { .. } => RbcPhaseStatementV1::Vote { + target: conflicting, + }, + RbcPhaseStatementV1::Ack { .. } => RbcPhaseStatementV1::Ack { + target: conflicting, + }, + _ => unreachable!(), + }; + record_phase(&mut model, 1, conflicting_statement); + } + let slot = model.rbc_slots.get(&(1, 0)).unwrap(); + assert_eq!(slot.vote_by_sender[&1], first); + assert_eq!(slot.ack_by_sender[&1], first); } #[test] @@ -2409,7 +3299,7 @@ mod tests { assert_eq!(receiver.delivered(0, 1), Some(first_ref)); assert_ne!(receiver.delivered(0, 1), Some(conflicting_ref)); let slot = receiver.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.echo_by_sender.get(&0), Some(&first_ref)); + assert_eq!(slot.echo_by_sender.get(&0), None); assert!(!slot.candidates.contains_key(&conflicting_ref)); } @@ -2636,14 +3526,14 @@ mod tests { let carrier = candidate( &committee, 1, - 6, - BlockReference::new_test(1, 5), + 66, + BlockReference::new_test(1, 65), vec![ - BlockReference::new_test(0, 5), - BlockReference::new_test(2, 5), + BlockReference::new_test(0, 65), + BlockReference::new_test(2, 65), ], Vec::new(), - 60, + 660, ) .unwrap(); let reference = carrier.reference(); @@ -2653,8 +3543,8 @@ mod tests { model.receive_authenticated(authenticated), Err(ModelError::FutureCarrierOutsideBuffer { current: 1, - maximum: 5, - actual: 6, + maximum: 65, + actual: 66, }) ); assert!(model.lifecycle(&reference).is_none()); @@ -2662,6 +3552,47 @@ mod tests { assert!(model.rbc_slots.is_empty()); } + #[test] + fn carrier_at_the_future_buffer_boundary_is_retained_but_cannot_advance() { + let committee = committee(4); + let mut model = model(Arc::clone(&committee), 0); + let carrier = candidate( + &committee, + 1, + 65, + BlockReference::new_test(1, 64), + vec![ + BlockReference::new_test(0, 64), + BlockReference::new_test(2, 64), + ], + Vec::new(), + 650, + ) + .unwrap(); + let reference = carrier.reference(); + + assert!( + model + .receive_authenticated(authenticate_for(&committee, &carrier, 0)) + .unwrap() + .is_empty() + ); + assert_eq!(model.local_carrier_round(), 1); + assert_eq!( + model.lifecycle(&reference), + Some(CarrierLifecycle { + authenticated: true, + admitted: false, + phase_batch_processed: true, + delivered: false, + certified_delivered: false, + data_available: false, + prefix_closed: false, + }) + ); + assert!(model.rbc_slots.is_empty()); + } + #[test] fn buffered_authenticated_carrier_is_promoted_when_window_opens() { let committee = committee(4); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 5c17e5ee..03dfa7d6 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -9,6 +9,7 @@ //! evaluates the explicit vote/no-vote evidence committed by those vertices. use std::{ + cmp::Reverse, collections::{BTreeMap, BTreeSet}, error::Error, fmt, @@ -56,6 +57,27 @@ pub enum ProjectionDecisionV1 { }, } +/// Immutable level-two evidence that allows an honest author to create the +/// next consensus vertex through the optimistic C1 pacemaker condition. +/// +/// Every returned reference is from consensus round `c - 1` and there is at +/// most one reference per author. A vote witness contains quorum stake voting +/// for one exact leader at `c - 2`, plus any caller-required own/leader parent +/// that is not already in that proof. A skip witness is the deterministic +/// union of the per-candidate negative-choice quorums required by the +/// direct-skip evaluator and those same required parents. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum C1StrongParentWitnessV1 { + Vote { + leader: ConsensusVertexReference, + parents: Vec, + }, + DirectSkip { + slot: LeaderSlotV1, + parents: Vec, + }, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub enum CertifiedProjectionError { CommitteeMismatch, @@ -89,11 +111,6 @@ pub enum CertifiedProjectionError { required: Option, actual: Option, }, - FrontierRegressesCommitted { - authority: AuthorityIndex, - committed: Option, - actual: Option, - }, StakeOverflow, InvalidLeaderSlot(LeaderSlotV1), MultipleCertifiedLeaderValues(LeaderSlotV1), @@ -241,6 +258,12 @@ impl CertifiedProjectionModel { Ok(()) } + pub(crate) fn is_data_available(&self, reference: BlockReference) -> bool { + self.carriers + .get(&reference) + .is_some_and(|state| state.data_available) + } + pub fn carrier_is_stored(&self, reference: BlockReference) -> bool { self.carriers.contains_key(&reference) } @@ -303,6 +326,15 @@ impl CertifiedProjectionModel { self.vertices.len() } + pub(crate) fn projected_stake_at_round(&self, round: RoundNumber) -> Stake { + self.vertices_at_round(round) + .map(|(reference, _)| reference.author()) + .collect::>() + .into_iter() + .filter_map(|authority| self.committee.get_stake(authority)) + .fold(0, Stake::saturating_add) + } + pub fn slot_values( &self, author: AuthorityIndex, @@ -329,6 +361,122 @@ impl CertifiedProjectionModel { .map(|projected| projected.vertex.leader_choice()) } + /// Return the exact C1 witness for creating consensus round `c`. + /// + /// Exact-vote witnesses consider every projected equivocation, then select + /// one deterministic matching value per author. Direct-skip witnesses use + /// one deterministic representative per author so their per-candidate + /// negative quorums have one compatible immutable union. Returning `None` + /// means C1 is not ready and the caller must wait or use a separately + /// justified C2/C3 fallback. + pub(crate) fn c1_strong_parent_witness( + &self, + consensus_round: RoundNumber, + required_parents: &[ConsensusVertexReference], + ) -> Result, CertifiedProjectionError> { + if consensus_round < 3 { + return Ok(None); + } + let voting_round = consensus_round - 1; + let slot = self.leader_slot(consensus_round - 2); + let representatives = self.deterministic_round_values(voting_round); + + let mut votes = BTreeMap::< + ConsensusVertexReference, + BTreeMap, + >::new(); + for (reference, _) in self.vertices_at_round(voting_round) { + if let Some(LeaderChoiceV1::Vote { leader }) = self.leader_choice(reference) { + votes + .entry(leader) + .or_default() + .entry(reference.author()) + .or_insert(reference); + } + } + let mut vote_witnesses = Vec::new(); + for (leader, voters) in votes { + if let Some(parents) = + self.frontier_fresh_quorum(voters.into_values(), required_parents)? + { + vote_witnesses.push((leader, parents)); + } + } + if vote_witnesses.len() > 1 { + return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues( + slot, + )); + } + if let Some((leader, parents)) = vote_witnesses.pop() { + return Ok(Some(C1StrongParentWitnessV1::Vote { leader, parents })); + } + + let candidates = self.slot_values(slot.author, slot.round); + let mut union = BTreeMap::new(); + if candidates.is_empty() { + let Some(parents) = + self.frontier_fresh_quorum(representatives.values().copied(), required_parents)? + else { + return Ok(None); + }; + for reference in parents { + union.insert(reference.author(), reference); + } + } else { + for candidate in candidates { + let negative = representatives.values().copied().filter(|reference| { + self.leader_choice(*reference) + .is_some_and(|choice| match choice { + LeaderChoiceV1::Vote { leader } => leader != candidate, + LeaderChoiceV1::NoVote { .. } => true, + }) + }); + let Some(parents) = self.lexicographic_quorum(negative)? else { + return Ok(None); + }; + for reference in parents { + union.insert(reference.author(), reference); + } + } + if !self.extend_with_required_parents(&mut union, required_parents) { + return Ok(None); + } + } + Ok(Some(C1StrongParentWitnessV1::DirectSkip { + slot, + parents: union.into_values().collect(), + })) + } + + /// Componentwise join of the exact effective frontiers inherited through + /// one immutable strong-parent set. Callers extend only their own + /// component from this base; copying the globally freshest closed + /// frontier would make every consensus vertex wait for unrelated + /// all-author delivery tails. + pub(crate) fn joined_strong_parent_frontier( + &self, + strong_parents: &[ConsensusVertexReference], + ) -> Result { + let mut parent_frontiers = Vec::with_capacity(strong_parents.len()); + for parent in strong_parents { + if parent.consensus_round() == 0 { + if parent.carrier() != carrier_genesis_reference(parent.author()) { + return Err(CertifiedProjectionError::InvalidGenesisStrongParent( + *parent, + )); + } + parent_frontiers.push(vec![None; self.committee.len()]); + continue; + } + let projected = self + .vertices + .get(parent) + .ok_or(CertifiedProjectionError::MissingStrongParent(*parent))?; + parent_frontiers.push(projected.effective_frontier.clone()); + } + self.join_frontiers(&parent_frontiers) + } + /// Project one optional consensus vertex if every stateful eligibility /// condition holds. Failure leaves the enclosing carrier untouched. pub fn try_project( @@ -489,22 +637,26 @@ impl CertifiedProjectionModel { } } - /// Record an externally selected committed anchor while enforcing exact - /// componentwise frontier monotonicity. The runtime committer remains out - /// of scope for this model. + /// Record an externally selected committed anchor and return the monotone + /// componentwise join of every committed anchor frontier. Consecutive + /// Starfish leaders need not be ancestors of one another, so their exact + /// frontiers may advance different authority components concurrently. + /// Logical anchor membership is therefore independent of whether this + /// particular anchor advances the accumulated output frontier. pub fn record_committed_anchor( &mut self, anchor: ConsensusVertexReference, - ) -> Result<(), CertifiedProjectionError> { + ) -> Result { let projected = self .vertices .get(&anchor) .ok_or(CertifiedProjectionError::MissingStrongParent(anchor))?; let frontier = projected.effective_frontier.clone(); - self.ensure_dominates_committed(&frontier)?; - self.committed_frontier = frontier; + let accumulated = self.committed_frontier.clone(); + let joined = self.join_frontiers(&[accumulated, frontier])?; + self.committed_frontier.clone_from(&joined); self.committed_anchors.insert(anchor); - Ok(()) + Ok(joined) } /// Decide an older leader from a later committed anchor. A reachable @@ -638,31 +790,65 @@ impl CertifiedProjectionModel { Ok(()) } - fn ensure_dominates_committed( + /// True iff `descendant` is the same exact prefix tip as `base`, or an + /// exact self-chain extension whose intermediate carrier headers are known. + fn is_exact_extension( &self, - frontier: &[Option], - ) -> Result<(), CertifiedProjectionError> { - for (index, (committed, actual)) in self - .committed_frontier - .iter() - .copied() - .zip(frontier.iter().copied()) - .enumerate() - { - if !self.is_exact_extension(committed, actual) { - return Err(CertifiedProjectionError::FrontierRegressesCommitted { - authority: index as AuthorityIndex, - committed, - actual, - }); - } + base: Option, + descendant: Option, + ) -> bool { + self.exact_extension_on_closed_prefix(base, descendant) + .unwrap_or_else(|| self.is_exact_extension_by_chain(base, descendant)) + } + + /// Resolve comparisons whose answer is already encoded by the exact + /// per-author closed prefix. Returning `None` preserves the historical + /// header-chain walk for staged forks and incomplete/non-closed tails. + fn exact_extension_on_closed_prefix( + &self, + base: Option, + descendant: Option, + ) -> Option { + let Some(descendant) = descendant else { + return Some(base.is_none()); + }; + if base.is_some_and(|base| base.authority != descendant.authority) { + return Some(false); + } + let base_round = base.map_or(0, |reference| reference.round); + if descendant.round < base_round { + return Some(false); + } + if base == Some(descendant) { + return Some(true); + } + if descendant.round == base_round { + return Some(match base { + Some(_) => false, + None => descendant == carrier_genesis_reference(descendant.authority), + }); + } + if !self.is_exact_closed_prefix_reference(descendant) { + return None; + } + match base { + None => Some(true), + Some(base) if self.is_exact_closed_prefix_reference(base) => Some(true), + Some(_) => None, } - Ok(()) } - /// True iff `descendant` is the same exact prefix tip as `base`, or an - /// exact self-chain extension whose intermediate carrier headers are known. - fn is_exact_extension( + fn is_exact_closed_prefix_reference(&self, reference: BlockReference) -> bool { + if reference.round == 0 { + return reference == carrier_genesis_reference(reference.authority); + } + self.closed_prefixes + .get(reference.authority as usize) + .and_then(|prefix| prefix.get((reference.round - 1) as usize)) + .is_some_and(|closed| *closed == reference) + } + + fn is_exact_extension_by_chain( &self, base: Option, descendant: Option, @@ -717,6 +903,215 @@ impl CertifiedProjectionModel { }) } + fn deterministic_round_values( + &self, + round: RoundNumber, + ) -> BTreeMap { + let mut by_author = BTreeMap::new(); + for (reference, _) in self.vertices_at_round(round) { + by_author.entry(reference.author()).or_insert(reference); + } + by_author + } + + fn lexicographic_quorum( + &self, + references: impl Iterator, + ) -> Result>, CertifiedProjectionError> { + let mut stake = 0u64; + let mut selected = Vec::new(); + for reference in references { + let author_stake = self + .committee + .get_stake(reference.author()) + .ok_or(CertifiedProjectionError::StakeOverflow)?; + stake = stake + .checked_add(author_stake) + .ok_or(CertifiedProjectionError::StakeOverflow)?; + selected.push(reference); + if stake >= self.committee.quorum_threshold() { + return Ok(Some(selected)); + } + } + Ok(None) + } + + /// Choose quorum evidence that maximizes new exact-prefix coverage without + /// increasing the parent budget of the former canonical-prefix selector. + /// + /// The baseline is the lexicographic quorum unioned with every required + /// parent. The greedy candidate set may use at most that many references. + /// Each choice must leave enough remaining slots to complete weighted + /// quorum stake; if freshness selection cannot do so, the known-valid + /// baseline is returned. Required references count toward proof stake only + /// when they are exact members of `references`. + pub(crate) fn frontier_fresh_quorum( + &self, + references: impl IntoIterator, + required: &[ConsensusVertexReference], + ) -> Result>, CertifiedProjectionError> { + let mut candidates = BTreeMap::::new(); + for reference in references { + candidates + .entry(reference.author()) + .and_modify(|existing| *existing = (*existing).min(reference)) + .or_insert(reference); + } + + let Some(baseline_proof) = self.lexicographic_quorum(candidates.values().copied())? else { + return Ok(None); + }; + let mut baseline = baseline_proof + .into_iter() + .map(|reference| (reference.author(), reference)) + .collect::>(); + if !self.extend_with_required_parents(&mut baseline, required) { + return Ok(None); + } + let parent_budget = baseline.len(); + + let mut selected = BTreeMap::new(); + if !self.extend_with_required_parents(&mut selected, required) { + return Ok(None); + } + let mut proof_stake = selected + .iter() + .filter(|(author, reference)| candidates.get(author) == Some(reference)) + .try_fold(0u64, |stake, (author, _)| { + stake + .checked_add( + self.committee + .get_stake(*author) + .ok_or(CertifiedProjectionError::StakeOverflow)?, + ) + .ok_or(CertifiedProjectionError::StakeOverflow) + })?; + let quorum = self.committee.quorum_threshold(); + let tie_origin: RoundNumber = candidates + .values() + .next() + .map_or(0, |reference| reference.consensus_round()) + % u32::try_from(self.committee.len()) + .unwrap_or(u32::MAX) + .max(1); + + while proof_stake < quorum && selected.len() < parent_budget { + let selected_references = selected.values().copied().collect::>(); + let base = match self.joined_strong_parent_frontier(&selected_references) { + Ok(base) => base, + Err(_) => return Ok(Some(baseline.into_values().collect())), + }; + let remaining_slots = parent_budget.saturating_sub(selected.len() + 1); + let mut best = None; + for (author, reference) in &candidates { + if selected.contains_key(author) { + continue; + } + let Some(projected) = self.vertices.get(reference) else { + continue; + }; + if self + .join_frontiers(&[base.clone(), projected.effective_frontier.clone()]) + .is_err() + { + continue; + } + let author_stake = self + .committee + .get_stake(*author) + .ok_or(CertifiedProjectionError::StakeOverflow)?; + let candidate_stake = proof_stake + .checked_add(author_stake) + .ok_or(CertifiedProjectionError::StakeOverflow)?; + let mut remaining_stakes: Vec = candidates + .keys() + .filter(|candidate_author| { + **candidate_author != *author && !selected.contains_key(*candidate_author) + }) + .map(|candidate_author| { + self.committee + .get_stake(*candidate_author) + .ok_or(CertifiedProjectionError::StakeOverflow) + }) + .collect::, _>>()?; + remaining_stakes.sort_unstable_by(|left: &Stake, right: &Stake| right.cmp(left)); + let maximum_completion = remaining_stakes + .into_iter() + .take(remaining_slots) + .try_fold(candidate_stake, |stake: Stake, next: Stake| { + stake + .checked_add(next) + .ok_or(CertifiedProjectionError::StakeOverflow) + })?; + if maximum_completion < quorum { + continue; + } + + let (advanced_components, round_advance) = base + .iter() + .zip(&projected.effective_frontier) + .fold((0usize, 0u64), |(components, advance), (base, tip)| { + let base_round = base.map_or(0, |reference| reference.round); + let tip_round = tip.map_or(0, |reference| reference.round); + let delta = tip_round.saturating_sub(base_round); + ( + components.saturating_add(usize::from(delta != 0)), + advance.saturating_add(u64::from(delta)), + ) + }); + let committee_size = u32::try_from(self.committee.len()) + .unwrap_or(u32::MAX) + .max(1); + let rotating_rank = + u32::from(*author).wrapping_add(committee_size - tie_origin) % committee_size; + let score = ( + advanced_components, + round_advance, + Reverse(rotating_rank), + Reverse(*reference), + ); + if best + .as_ref() + .is_none_or(|(best_score, _, _)| score > *best_score) + { + best = Some((score, *author, *reference)); + } + } + let Some((_, author, reference)) = best else { + return Ok(Some(baseline.into_values().collect())); + }; + selected.insert(author, reference); + proof_stake = proof_stake + .checked_add( + self.committee + .get_stake(author) + .ok_or(CertifiedProjectionError::StakeOverflow)?, + ) + .ok_or(CertifiedProjectionError::StakeOverflow)?; + } + + if proof_stake < quorum { + return Ok(Some(baseline.into_values().collect())); + } + Ok(Some(selected.into_values().collect())) + } + + fn extend_with_required_parents( + &self, + selected: &mut BTreeMap, + required: &[ConsensusVertexReference], + ) -> bool { + for reference in required { + if selected + .insert(reference.author(), *reference) + .is_some_and(|existing| existing != *reference) + { + return false; + } + } + true + } + fn voter_authors( &self, slot: LeaderSlotV1, @@ -813,7 +1208,7 @@ impl CertifiedProjectionModel { } #[cfg(test)] - fn inject_projected_for_test( + pub(crate) fn inject_projected_for_test( &mut self, reference: ConsensusVertexReference, strong_parents: Vec, @@ -843,6 +1238,139 @@ impl CertifiedProjectionModel { } } +/// Independent planning-only projection over promised, data-available carrier +/// prefixes. +/// +/// This wrapper deliberately exposes no decision, committed-anchor, or +/// committed-frontier API. Its inner model reuses the exact carrier-chain, +/// strong-parent, and frontier validation of the certified projection, but +/// interprets that private plane's delivery latch as `DeliveryPromised`. +/// Consequently its contiguous prefixes and projected vertices can run ahead +/// of certification without becoming output authority. +#[derive(Clone)] +pub(crate) struct PromisedProjectionModel { + inner: CertifiedProjectionModel, +} + +impl PromisedProjectionModel { + pub(crate) fn from_committee_context(committee: RbcDagCommitteeContextV1) -> Self { + Self { + inner: CertifiedProjectionModel::from_committee_context(committee), + } + } + + pub(crate) fn stage_carrier( + &mut self, + candidate: CandidateCarrierV1, + ) -> Result<(), CertifiedProjectionError> { + self.inner.stage_carrier(candidate) + } + + /// Mark one exact carrier promised. This advances only the planner's + /// private contiguous prefix once local DA is also established. + pub(crate) fn mark_promised( + &mut self, + reference: BlockReference, + ) -> Result<(), CertifiedProjectionError> { + self.inner.mark_delivered(reference) + } + + pub(crate) fn mark_data_available( + &mut self, + reference: BlockReference, + ) -> Result<(), CertifiedProjectionError> { + self.inner.mark_data_available(reference) + } + + #[cfg(test)] + pub(crate) fn is_data_available(&self, reference: BlockReference) -> bool { + self.inner.is_data_available(reference) + } + + pub(crate) fn carrier_is_stored(&self, reference: BlockReference) -> bool { + self.inner.carrier_is_stored(reference) + } + + #[cfg(test)] + pub(crate) fn promised_tip(&self, authority: AuthorityIndex) -> Option { + self.inner.closed_tip(authority) + } + + #[cfg(test)] + pub(crate) fn is_projected(&self, reference: ConsensusVertexReference) -> bool { + self.inner.is_projected(reference) + } + + #[cfg(test)] + pub(crate) fn projected_vertex( + &self, + reference: ConsensusVertexReference, + ) -> Option<&ConsensusVertexV1> { + self.inner.projected_vertex(reference) + } + + #[cfg(test)] + pub(crate) fn effective_frontier( + &self, + reference: ConsensusVertexReference, + ) -> Option<&[Option]> { + self.inner.effective_frontier(reference) + } + + pub(crate) fn projected_values_at_round( + &self, + round: RoundNumber, + ) -> Vec { + self.inner.projected_values_at_round(round) + } + + pub(crate) fn projected_stake_at_round(&self, round: RoundNumber) -> Stake { + self.inner.projected_stake_at_round(round) + } + + pub(crate) fn c1_strong_parent_witness( + &self, + consensus_round: RoundNumber, + required_parents: &[ConsensusVertexReference], + ) -> Result, CertifiedProjectionError> { + self.inner + .c1_strong_parent_witness(consensus_round, required_parents) + } + + pub(crate) fn frontier_fresh_quorum( + &self, + references: impl IntoIterator, + required: &[ConsensusVertexReference], + ) -> Result>, CertifiedProjectionError> { + self.inner.frontier_fresh_quorum(references, required) + } + + pub(crate) fn joined_strong_parent_frontier( + &self, + strong_parents: &[ConsensusVertexReference], + ) -> Result { + self.inner.joined_strong_parent_frontier(strong_parents) + } + + pub(crate) fn try_project( + &mut self, + carrier_reference: BlockReference, + ) -> Result { + self.inner.try_project(carrier_reference) + } + + #[cfg(test)] + pub(crate) fn inject_projected_for_test( + &mut self, + reference: ConsensusVertexReference, + strong_parents: Vec, + leader_choice: LeaderChoiceV1, + ) { + self.inner + .inject_projected_for_test(reference, strong_parents, leader_choice); + } +} + #[cfg(test)] mod tests { use super::*; @@ -923,6 +1451,57 @@ mod tests { reference } + fn close_author_history( + model: &mut CertifiedProjectionModel, + author: AuthorityIndex, + rounds: RoundNumber, + ) -> Vec { + let mut result = Vec::with_capacity(rounds as usize); + let mut own_previous = carrier_genesis_reference(author); + for round in 1..=rounds { + let previous = model + .committee + .authorities() + .map(|authority| { + if authority == author { + own_previous + } else if round == 1 { + carrier_genesis_reference(authority) + } else { + reference( + authority, + round - 1, + (authority as u8).wrapping_mul(31).wrapping_add(round as u8), + ) + } + }) + .collect::>(); + let carrier = candidate( + &model.committee, + author, + round, + &previous, + None, + (author as u8).wrapping_mul(53).wrapping_add(round as u8), + ); + own_previous = clean(model, carrier); + result.push(own_previous); + } + result + } + + fn assert_exact_extension_matches_chain_oracle( + model: &CertifiedProjectionModel, + base: Option, + descendant: Option, + ) { + assert_eq!( + model.is_exact_extension(base, descendant), + model.is_exact_extension_by_chain(base, descendant), + "base={base:?}, descendant={descendant:?}", + ); + } + fn first_consensus_round( model: &mut CertifiedProjectionModel, ) -> (Vec, Vec) { @@ -1009,6 +1588,229 @@ mod tests { assert_eq!(&effective[1..], expected.as_slice()); } + #[test] + fn closed_prefix_exact_extension_fast_path_matches_chain_oracle() { + const LAST_ROUND: RoundNumber = 512; + + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let closed = close_author_history(&mut model, 0, LAST_ROUND); + let genesis = carrier_genesis_reference(0); + let closed_samples = [closed[0], closed[16], closed[255], closed[511]]; + + assert_eq!( + model.exact_extension_on_closed_prefix(None, Some(closed[511])), + Some(true) + ); + assert_eq!( + model.exact_extension_on_closed_prefix(Some(closed[16]), Some(closed[511])), + Some(true) + ); + for base in [None, Some(genesis)] + .into_iter() + .chain(closed_samples.into_iter().map(Some)) + { + for descendant in [None, Some(genesis)] + .into_iter() + .chain(closed_samples.into_iter().map(Some)) + { + assert_exact_extension_matches_chain_oracle(&model, base, descendant); + } + } + + let same_round_fork = reference(0, LAST_ROUND, 0xF1); + let cross_author = reference(1, LAST_ROUND, 0xF2); + let missing_descendant = reference(0, LAST_ROUND + 4, 0xF3); + let invalid_genesis = reference(0, 0, 0xF4); + for (base, descendant) in [ + (Some(closed[511]), Some(same_round_fork)), + (Some(closed[16]), Some(same_round_fork)), + (Some(closed[16]), Some(cross_author)), + (Some(closed[511]), Some(missing_descendant)), + (None, Some(invalid_genesis)), + (Some(closed[511]), None), + (None, None), + ] { + assert_exact_extension_matches_chain_oracle(&model, base, descendant); + } + + // A fully staged non-closed tail remains an exact known extension and + // therefore exercises the historical chain-walk fallback. + let staged_previous = model + .committee + .authorities() + .map(|authority| { + if authority == 0 { + closed[511] + } else { + reference(authority, LAST_ROUND, 0xA0 + authority as u8) + } + }) + .collect::>(); + let staged = candidate( + &model.committee, + 0, + LAST_ROUND + 1, + &staged_previous, + None, + 0xF5, + ); + let staged_reference = staged.reference(); + model.stage_carrier(staged).unwrap(); + let mut staged_child_previous = staged_previous; + staged_child_previous[0] = staged_reference; + for (authority, previous) in staged_child_previous.iter_mut().enumerate().skip(1) { + *previous = reference( + authority as AuthorityIndex, + LAST_ROUND + 1, + 0xB0 + authority as u8, + ); + } + let staged_child = candidate( + &model.committee, + 0, + LAST_ROUND + 2, + &staged_child_previous, + None, + 0xF6, + ); + let staged_child_reference = staged_child.reference(); + model.stage_carrier(staged_child).unwrap(); + assert_eq!( + model + .exact_extension_on_closed_prefix(Some(closed[511]), Some(staged_child_reference),), + None + ); + assert_exact_extension_matches_chain_oracle( + &model, + Some(closed[511]), + Some(staged_child_reference), + ); + assert!(model.is_exact_extension(Some(closed[511]), Some(staged_child_reference))); + + // A staged descendant whose intermediate own-prev is absent also + // falls back, then rejects exactly as the prior implementation did. + let missing_previous = reference(0, LAST_ROUND + 2, 0xF7); + let missing_chain_previous = model + .committee + .authorities() + .map(|authority| { + if authority == 0 { + missing_previous + } else { + reference(authority, LAST_ROUND + 2, 0xC0 + authority as u8) + } + }) + .collect::>(); + let missing_chain = candidate( + &model.committee, + 0, + LAST_ROUND + 3, + &missing_chain_previous, + None, + 0xF8, + ); + let missing_chain_reference = missing_chain.reference(); + model.stage_carrier(missing_chain).unwrap(); + assert_exact_extension_matches_chain_oracle( + &model, + Some(closed[511]), + Some(missing_chain_reference), + ); + assert!(!model.is_exact_extension(Some(closed[511]), Some(missing_chain_reference))); + } + + #[test] + fn closed_prefix_fast_path_preserves_join_dominance_and_committed_frontiers() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let histories = (0..4) + .map(|authority| close_author_history(&mut model, authority, 32)) + .collect::>(); + let first = vec![ + Some(histories[0][7]), + None, + Some(histories[2][11]), + Some(histories[3][3]), + ]; + let second = vec![ + Some(histories[0][15]), + Some(histories[1][5]), + Some(histories[2][2]), + None, + ]; + let joined = vec![ + Some(histories[0][15]), + Some(histories[1][5]), + Some(histories[2][11]), + Some(histories[3][3]), + ]; + + assert_eq!( + model + .join_frontiers(&[first.clone(), second.clone()]) + .unwrap(), + joined + ); + model.ensure_dominates_parent(&joined, &first).unwrap(); + model.ensure_dominates_parent(&joined, &second).unwrap(); + assert!(matches!( + model.ensure_dominates_parent(&first, &second), + Err(CertifiedProjectionError::FrontierDoesNotDominateParent { authority: 0, .. }) + )); + + let first_anchor = consensus_reference(0, 40, 0xD1); + let second_anchor = consensus_reference(1, 41, 0xD2); + for anchor in [first_anchor, second_anchor] { + model.inject_projected_for_test( + anchor, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 39, + }, + ); + } + model + .vertices + .get_mut(&first_anchor) + .unwrap() + .effective_frontier + .clone_from(&first); + model + .vertices + .get_mut(&second_anchor) + .unwrap() + .effective_frontier + .clone_from(&second); + assert_eq!( + model + .joined_strong_parent_frontier(&[first_anchor, second_anchor]) + .unwrap(), + joined + ); + assert_eq!(model.record_committed_anchor(first_anchor).unwrap(), first); + assert_eq!( + model.record_committed_anchor(second_anchor).unwrap(), + joined + ); + assert!(model.committed_frontier_dominates(&first)); + assert!(model.committed_frontier_dominates(&second)); + assert!(model.committed_frontier_dominates(&joined)); + + let fork = Some(reference(2, histories[2][11].round, 0xD3)); + let mut forked = joined.clone(); + forked[2] = fork; + assert!(matches!( + model.join_frontiers(&[joined.clone(), forked.clone()]), + Err(CertifiedProjectionError::ParentFrontierFork { authority: 2, .. }) + )); + assert!(!model.committed_frontier_dominates(&forked)); + let mut beyond_commit = joined; + beyond_commit[3] = Some(histories[3][31]); + assert!(!model.committed_frontier_dominates(&beyond_commit)); + } + #[test] fn omission_and_same_round_fork_do_not_pass_frontier_checks() { let committee = Committee::new_test(vec![1; 4]); @@ -1106,7 +1908,7 @@ mod tests { } #[test] - fn late_vertex_remains_visible_but_cannot_become_a_regressing_anchor() { + fn concurrent_committed_anchor_frontiers_accumulate_by_exact_component_join() { let committee = Committee::new_test(vec![1; 4]); let mut model = CertifiedProjectionModel::new(committee).unwrap(); let (carriers, parents) = first_consensus_round(&mut model); @@ -1138,12 +1940,23 @@ mod tests { let regressing_carrier = clean(&mut model, regressing); let regressing_vertex = model.try_project(regressing_carrier).unwrap(); assert!(model.is_projected(regressing_vertex)); - assert!(matches!( - model.record_committed_anchor(regressing_vertex), - Err(CertifiedProjectionError::FrontierRegressesCommitted { authority: 0, .. }) - )); + let accumulated = model.record_committed_anchor(regressing_vertex).unwrap(); + assert_eq!( + accumulated, + vec![ + Some(anchor_carrier), + Some(carriers[1]), + Some(regressing_carrier), + Some(carriers[3]), + ] + ); + assert!(model.is_committed_anchor(anchor_vertex)); + assert!(model.is_committed_anchor(regressing_vertex)); assert!( - !model + model.committed_frontier_dominates(model.effective_frontier(anchor_vertex).unwrap()) + ); + assert!( + model .committed_frontier_dominates(model.effective_frontier(regressing_vertex).unwrap()) ); } @@ -1419,6 +2232,348 @@ mod tests { ); } + #[test] + fn c1_waits_for_an_exact_vote_quorum_and_returns_only_its_witness() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); + let slot = model.leader_slot(1); + let leader = round_one_vertices[slot.author as usize]; + let frontier = round_one_carriers + .iter() + .copied() + .map(Some) + .collect::>(); + + let choices = [ + LeaderChoiceV1::Vote { leader }, + LeaderChoiceV1::Vote { leader }, + LeaderChoiceV1::Vote { leader }, + LeaderChoiceV1::NoVote { + leader_author: slot.author, + leader_round: slot.round, + }, + ]; + let mut projected = BTreeMap::new(); + for author in [0, 1, 3] { + let parents = if author == 3 { + round_one_vertices + .iter() + .copied() + .filter(|parent| parent.author() != slot.author) + .collect() + } else { + round_one_vertices.clone() + }; + let carrier = candidate( + &model.committee, + author, + 2, + &round_one_carriers, + Some(ConsensusVertexV1::new( + 2, + parents, + frontier.clone(), + choices[author as usize], + )), + 0xD0 + author as u8, + ); + let carrier = clean(&mut model, carrier); + projected.insert(author, model.try_project(carrier).unwrap()); + } + assert_eq!(model.c1_strong_parent_witness(3, &[]).unwrap(), None); + + let author = 2; + let carrier = candidate( + &model.committee, + author, + 2, + &round_one_carriers, + Some(ConsensusVertexV1::new( + 2, + round_one_vertices, + frontier, + choices[author as usize], + )), + 0xD0 + author as u8, + ); + let carrier = clean(&mut model, carrier); + projected.insert(author, model.try_project(carrier).unwrap()); + + let witness = model + .c1_strong_parent_witness(3, &[]) + .unwrap() + .expect("the third exact vote completes C1"); + assert_eq!( + witness, + C1StrongParentWitnessV1::Vote { + leader, + parents: [0, 1, 2] + .into_iter() + .map(|author| projected[&author]) + .collect(), + } + ); + let C1StrongParentWitnessV1::Vote { parents, .. } = witness else { + panic!("the exact vote quorum must return a vote witness"); + }; + assert!(!parents.iter().any(|parent| parent.author() == 3)); + } + + #[test] + fn c1_exact_vote_witness_is_not_hidden_by_a_smaller_byzantine_equivocation() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (_, round_one_vertices) = first_consensus_round(&mut model); + let slot = model.leader_slot(1); + let leader = round_one_vertices[slot.author as usize]; + let no_vote = LeaderChoiceV1::NoVote { + leader_author: slot.author, + leader_round: slot.round, + }; + + let byzantine_no_vote = consensus_reference(0, 2, 0x10); + let byzantine_vote = consensus_reference(0, 2, 0xF0); + assert!(byzantine_no_vote < byzantine_vote); + model.inject_projected_for_test(byzantine_no_vote, Vec::new(), no_vote); + model.inject_projected_for_test( + byzantine_vote, + Vec::new(), + LeaderChoiceV1::Vote { leader }, + ); + + let voter_one = consensus_reference(1, 2, 0x21); + let voter_two = consensus_reference(2, 2, 0x22); + let non_voter = consensus_reference(3, 2, 0x23); + for voter in [voter_one, voter_two] { + model.inject_projected_for_test(voter, Vec::new(), LeaderChoiceV1::Vote { leader }); + } + model.inject_projected_for_test(non_voter, Vec::new(), no_vote); + + assert_eq!( + model.c1_strong_parent_witness(3, &[]).unwrap(), + Some(C1StrongParentWitnessV1::Vote { + leader, + parents: vec![byzantine_vote, voter_one, voter_two], + }) + ); + } + + #[test] + fn joined_parent_frontier_omits_unrelated_fresh_closed_tips() { + let committee = Committee::new_test(vec![1; 4]); + let mut model = CertifiedProjectionModel::new(committee).unwrap(); + let (carriers, vertices) = first_consensus_round(&mut model); + assert_eq!( + model.closed_frontier(), + carriers.iter().copied().map(Some).collect::>() + ); + + let joined = model.joined_strong_parent_frontier(&vertices[..3]).unwrap(); + assert_eq!( + joined, + vec![ + Some(carriers[0]), + Some(carriers[1]), + Some(carriers[2]), + None + ] + ); + } + + #[test] + fn frontier_fresh_quorum_preserves_weighted_stake_and_parent_budget() { + let committee = Committee::new_test(vec![4, 3, 2, 1, 1]); + let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let previous = previous_carriers(&committee, 0); + let mut vertices = Vec::new(); + for author in committee.authorities() { + let carrier = candidate(&committee, author, 1, &previous, None, 0x70 + author as u8); + let carrier_reference = carrier.reference(); + model.stage_carrier(carrier).unwrap(); + let vertex = ConsensusVertexReference::new(carrier_reference, 1); + model.inject_projected_for_test( + vertex, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }, + ); + model.vertices.get_mut(&vertex).unwrap().effective_frontier[author as usize] = + Some(carrier_reference); + vertices.push(vertex); + } + + // The previous lexicographic proof used authors 0, 1, and 2; adding + // required author 4 gave a four-parent hard budget. + let selected = model + .frontier_fresh_quorum(vertices.iter().copied(), &[vertices[4]]) + .unwrap() + .unwrap(); + let selected_stake = selected + .iter() + .map(|reference| committee.get_stake(reference.author()).unwrap()) + .sum::(); + assert!(selected_stake >= committee.quorum_threshold()); + assert!(selected.contains(&vertices[4])); + assert!(selected.len() <= 4); + assert_eq!( + selected + .iter() + .map(|reference| reference.author()) + .collect::>() + .len(), + selected.len(), + "one equivocating author must never contribute stake twice" + ); + } + + fn simulated_parent_frontier_inclusion_lags(frontier_fresh: bool) -> Vec> { + const N: usize = 10; + const ROUNDS: RoundNumber = 30; + + let committee = Committee::new_test(vec![1; N]); + let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); + let mut previous = previous_carriers(&committee, 0); + let mut vertices_by_round = vec![Vec::new(); ROUNDS as usize + 1]; + + for round in 1..=ROUNDS { + let mut carriers = Vec::with_capacity(N); + for author in committee.authorities() { + let marker = ((round as usize * 17 + author as usize) % 251) as u8; + let carrier = candidate(&committee, author, round, &previous, None, marker); + let reference = carrier.reference(); + model.stage_carrier(carrier).unwrap(); + carriers.push(reference); + } + + let mut round_vertices = Vec::with_capacity(N); + for author in committee.authorities() { + let mut effective_frontier = vec![None; N]; + if round > 1 { + let parent_values = &vertices_by_round[round as usize - 1]; + let leader = parent_values[committee.elect_leader(round - 1) as usize]; + let own = parent_values[author as usize]; + let required = [own, leader]; + let strong_parents = if frontier_fresh { + model + .frontier_fresh_quorum(parent_values.iter().copied(), &required) + .unwrap() + .unwrap() + } else { + let mut selected = BTreeMap::new(); + if round >= 3 { + let mut proof_stake = 0; + for parent in parent_values { + selected.insert(parent.author(), *parent); + proof_stake += committee.get_stake(parent.author()).unwrap(); + if proof_stake >= committee.quorum_threshold() { + break; + } + } + for parent in required { + selected.insert(parent.author(), parent); + } + } else { + for parent in required { + selected.insert(parent.author(), parent); + } + let mut stake = selected + .keys() + .map(|authority| committee.get_stake(*authority).unwrap()) + .sum::(); + for parent in parent_values { + if stake >= committee.quorum_threshold() { + break; + } + if selected.insert(parent.author(), *parent).is_none() { + stake += committee.get_stake(parent.author()).unwrap(); + } + } + } + selected.into_values().collect() + }; + effective_frontier = model + .joined_strong_parent_frontier(&strong_parents) + .unwrap(); + } + effective_frontier[author as usize] = Some(carriers[author as usize]); + let vertex = ConsensusVertexReference::new(carriers[author as usize], round); + model.inject_projected_for_test( + vertex, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: committee.elect_leader(round.saturating_sub(1)), + leader_round: round.saturating_sub(1), + }, + ); + model.vertices.get_mut(&vertex).unwrap().effective_frontier = effective_frontier; + round_vertices.push(vertex); + } + vertices_by_round[round as usize] = round_vertices; + previous = carriers; + } + + let mut lags = vec![Vec::new(); N]; + // Leave a complete leader rotation at the tail so the intentionally + // biased baseline also has time to include every high-index author. + for application_round in 3..=ROUNDS - 12 { + for author in committee.authorities() { + let first_anchor = (application_round..=ROUNDS - 2) + .find(|anchor_round| { + let leader = vertices_by_round[*anchor_round as usize] + [committee.elect_leader(*anchor_round) as usize]; + model.effective_frontier(leader).unwrap()[author as usize] + .is_some_and(|tip| tip.round >= application_round) + }) + .expect("every healthy application prefix must reach a committed leader"); + lags[author as usize].push(first_anchor - application_round); + } + } + lags + } + + #[test] + fn frontier_fresh_quorums_remove_lexicographic_author_inclusion_tail() { + let lexicographic = simulated_parent_frontier_inclusion_lags(false); + let low_author_average = lexicographic[..7] + .iter() + .flatten() + .copied() + .map(u64::from) + .sum::() as f64 + / lexicographic[..7].iter().map(Vec::len).sum::() as f64; + let high_author_average = lexicographic[7..] + .iter() + .flatten() + .copied() + .map(u64::from) + .sum::() as f64 + / lexicographic[7..].iter().map(Vec::len).sum::() as f64; + assert!( + high_author_average > low_author_average + 2.0, + "the regression fixture must expose the former 0..6/7..9 tail" + ); + + let fresh = simulated_parent_frontier_inclusion_lags(true); + let maximum_lag = fresh.iter().flatten().copied().max().unwrap(); + assert!( + maximum_lag <= 2, + "a healthy n=10 frontier should enter a leader within two logical rounds, got {fresh:?}" + ); + let per_author_average = fresh + .iter() + .map(|lags| lags.iter().copied().map(u64::from).sum::() as f64 / lags.len() as f64) + .collect::>(); + let minimum = per_author_average.iter().copied().reduce(f64::min).unwrap(); + let maximum = per_author_average.iter().copied().reduce(f64::max).unwrap(); + assert!( + maximum - minimum < 1.0, + "author skew remains: {per_author_average:?}" + ); + } + #[test] fn direct_skip_uses_clean_projected_explicit_negative_choices() { let committee = Committee::new_test(vec![1; 4]); @@ -1446,7 +2601,7 @@ mod tests { leader_round: slot.round, }; let voter_choices = vec![no_vote, LeaderChoiceV1::Vote { leader }, no_vote, no_vote]; - project_complete_round( + let (_, voters) = project_complete_round( &mut model, 2, &round_one_carriers, @@ -1459,6 +2614,13 @@ mod tests { model.direct_decision(slot).unwrap(), ProjectionDecisionV1::DirectSkip { slot } ); + assert_eq!( + model.c1_strong_parent_witness(3, &[]).unwrap(), + Some(C1StrongParentWitnessV1::DirectSkip { + slot, + parents: [voters[0], voters[2], voters[3]].to_vec(), + }) + ); } fn indirect_graph( @@ -1601,13 +2763,13 @@ mod tests { ); let anchor_carrier = clean(&mut model, anchor_carrier); let anchor = model.try_project(anchor_carrier).unwrap(); - model.record_committed_anchor(anchor).unwrap(); (model, slot, leader, anchor) } #[test] fn later_committed_anchor_drives_indirect_commit_or_skip() { - let (commit_model, slot, leader, commit_anchor) = indirect_graph(true); + let (mut commit_model, slot, leader, commit_anchor) = indirect_graph(true); + commit_model.record_committed_anchor(commit_anchor).unwrap(); assert_eq!( commit_model.indirect_decision(slot, commit_anchor).unwrap(), ProjectionDecisionV1::IndirectCommit { @@ -1616,7 +2778,8 @@ mod tests { } ); - let (skip_model, slot, _, skip_anchor) = indirect_graph(false); + let (mut skip_model, slot, _, skip_anchor) = indirect_graph(false); + skip_model.record_committed_anchor(skip_anchor).unwrap(); assert_eq!( skip_model.indirect_decision(slot, skip_anchor).unwrap(), ProjectionDecisionV1::IndirectSkip { @@ -1625,4 +2788,61 @@ mod tests { } ); } + + #[test] + fn already_dominated_anchor_identity_remains_usable_for_indirect_decision() { + let (mut model, slot, leader, anchor) = indirect_graph(true); + let anchor_frontier = model.effective_frontier(anchor).unwrap().to_vec(); + let anchor_previous = model + .carriers + .get(&anchor.carrier()) + .unwrap() + .candidate + .header() + .own_prev(); + let previous = model + .committee + .authorities() + .map(|authority| { + if authority == anchor.author() { + anchor_previous + } else { + model.closed_tip(authority).unwrap() + } + }) + .collect::>(); + let extension = candidate(&model.committee, 1, 4, &previous, None, 0xB0); + let extension = clean(&mut model, extension); + + let dominating = consensus_reference(1, 5, 0xB1); + model.inject_projected_for_test( + dominating, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: model.committee.elect_leader(4), + leader_round: 4, + }, + ); + let mut dominating_frontier = anchor_frontier; + dominating_frontier[1] = Some(extension); + model + .vertices + .get_mut(&dominating) + .unwrap() + .effective_frontier + .clone_from(&dominating_frontier); + assert_eq!( + model.record_committed_anchor(dominating).unwrap(), + dominating_frontier + ); + assert_eq!( + model.record_committed_anchor(anchor).unwrap(), + dominating_frontier + ); + assert!(model.is_committed_anchor(anchor)); + assert_eq!( + model.indirect_decision(slot, anchor).unwrap(), + ProjectionDecisionV1::IndirectCommit { leader, anchor } + ); + } } diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 5caa8c7a..02699c61 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -25,22 +25,27 @@ use crate::{ LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, carrier_genesis_reference, journal::{IngressProvenanceV1, JournalErrorV1, JournalEventV1, WriteAheadJournalV1}, - model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, + model::{ + DeliveryPromiseBasisV1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, ModelError, + ModelInputRecord, ModelTraceEvent, RbcDagModel, + }, projection::{ - CertifiedProjectionError, CertifiedProjectionModel, LeaderSlotV1, ProjectionDecisionV1, + C1StrongParentWitnessV1, CertifiedProjectionError, CertifiedProjectionModel, + LeaderSlotV1, ProjectionDecisionV1, PromisedProjectionModel, }, storage::{ MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, ShadowWalSummaryV1, ShadowWalSyncPolicyV1, ShadowWalV1, }, }, + store::RbcDagFrontierReceipt, types::{ AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, TimestampNs, }, }; -const RAW_RECORD_MAGIC: &[u8; 4] = b"SRD3"; +const RAW_RECORD_MAGIC: &[u8; 4] = b"SRD5"; const RAW_RECORD_VERSION_V1: u8 = 1; const RAW_RECORD_HEADER_SIZE: usize = 80; @@ -62,14 +67,19 @@ const TRACE_DELIVERY_LOCKED: u8 = 0x05; const TRACE_EFFECT: u8 = 0x06; const TRACE_CONSENSUS_SLOT_LOCKED: u8 = 0x07; const TRACE_LEADER_CHOICE_LOCKED: u8 = 0x08; +const TRACE_DELIVERY_PROMISE_LOCKED: u8 = 0x09; const EFFECT_NEED_CARRIER: u8 = 0x00; const EFFECT_DELIVERED: u8 = 0x01; const EFFECT_PREFIX_ADVANCED: u8 = 0x02; const EFFECT_CARRIER_ROUND_ADVANCED: u8 = 0x03; +const EFFECT_DELIVERY_PROMISED: u8 = 0x04; const PHASE_ECHO: u8 = 0x00; const PHASE_READY: u8 = 0x01; +// Phase codes are append-only because they are persisted in the shadow WAL. +const PHASE_VOTE: u8 = 0x02; +const PHASE_ACK: u8 = 0x03; const PROVENANCE_DIRECT: u8 = 0x00; const PROVENANCE_RELAYED: u8 = 0x01; @@ -79,6 +89,10 @@ const PROVENANCE_RELAYED: u8 = 0x01; /// protocol must derive pruning from a certified/committed watermark instead. /// Exact RBC recovery requests are exempt from this prototype guard. const SHADOW_BENCHMARK_UNSOLICITED_RETENTION_WINDOW_ROUNDS_V1: RoundNumber = 64; +/// A restart may replay only a bounded committed-frontier suffix into Core. +/// Larger gaps require a future checkpoint-transfer protocol rather than an +/// unbounded startup event burst. +pub(crate) const MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1: usize = 64; /// Local authentication material owned by exactly one shadow core. /// @@ -141,6 +155,10 @@ pub(crate) enum ShadowIngressDispositionV1 { Authenticated, CandidateRetained, IgnoredDuplicateConflictOrStale, + /// An unsolicited carrier was beyond the bounded future-retention + /// window. It was discarded before authentication and without WAL/model + /// mutation; exact current-round synchronization remains available. + IgnoredFutureOutsideBuffer, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -223,16 +241,77 @@ pub(crate) enum ShadowDeliveryComparisonV1 { }, } +/// Cheap, non-consensus physical-round observation for one first-committed +/// application. Unlike the discarded logical-ancestry diagnostic, this is +/// computed once at output and adds no work to projection. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct CommittedApplicationDiagnosticV1 { + pub(crate) physical_carrier_round_delta: i64, +} + +impl CommittedApplicationDiagnosticV1 { + fn new(anchor: ConsensusVertexReference, enclosing_carrier: BlockReference) -> Self { + Self { + physical_carrier_round_delta: i64::from(anchor.carrier().round) + - i64::from(enclosing_carrier.round), + } + } +} + /// Deterministic application output unlocked by one newly committed clean /// projected anchor. Carrier references remain available for audit while the /// application headers are already deduplicated and sorted by their carrier /// position in the exact committed frontier delta. #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CommittedFrontierDeltaV1 { + /// Monotone one-based output position, independent of the possibly + /// regressing logical round of `anchor`. + pub(crate) output_sequence: RoundNumber, pub(crate) anchor: ConsensusVertexReference, pub(crate) frontier: Vec>, pub(crate) carriers: Vec, pub(crate) applications: Vec, + /// Diagnostic-only records in one-to-one order with `applications`. + pub(crate) application_diagnostics: Vec, +} + +/// Runtime-only handoff between Core's compact durable receipt and the +/// authoritative carrier WAL. Application references are reconstructed from +/// the CommitData atomically stored under `receipt.carrier_anchor`; a missing +/// CommitData value therefore denotes an exact control-only frontier. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RbcDagFrontierRecoveryCursorV1 { + pub(crate) receipt: RbcDagFrontierReceipt, + pub(crate) application_references: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ProjectionHolReasonV1 { + InsufficientLookahead, + DirectEvidencePending, + AwaitingIndirectAnchor, + Ready, +} + +impl ProjectionHolReasonV1 { + pub(crate) const fn metric_label(self) -> &'static str { + match self { + Self::InsufficientLookahead => "insufficient_lookahead", + Self::DirectEvidencePending => "direct_evidence_pending", + Self::AwaitingIndirectAnchor => "awaiting_indirect_anchor", + Self::Ready => "ready", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ProjectionRuntimeSnapshotV1 { + pub(crate) pending_candidates: usize, + pub(crate) highest_projected_round: RoundNumber, + pub(crate) next_undecided_round: RoundNumber, + pub(crate) next_undecided_projected_stake: Stake, + pub(crate) last_committed_round: RoundNumber, + pub(crate) hol_reason: ProjectionHolReasonV1, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -240,6 +319,7 @@ pub(crate) struct ShadowOpenReportV1 { replayed_batches: u64, discarded_tail_bytes: u64, recovery_effects: Vec, + recovered_committed_frontiers: Vec, } impl ShadowOpenReportV1 { @@ -256,6 +336,24 @@ impl ShadowOpenReportV1 { pub(crate) fn recovery_effects(&self) -> &[ModelEffect] { &self.recovery_effects } + + /// Strictly newer authoritative WAL output that Core has not durably + /// acknowledged yet. Observational opens always return an empty suffix. + pub(crate) fn recovered_committed_frontiers(&self) -> &[CommittedFrontierDeltaV1] { + &self.recovered_committed_frontiers + } + + #[cfg(test)] + pub(crate) fn with_recovered_committed_frontiers_for_test( + recovered_committed_frontiers: Vec, + ) -> Self { + Self { + replayed_batches: 1, + discarded_tail_bytes: 0, + recovery_effects: Vec::new(), + recovered_committed_frontiers, + } + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -276,6 +374,7 @@ pub(crate) enum ShadowCodecErrorV1 { InvalidProvenance(u8), InvalidPhase(u8), InvalidLeaderChoice(u8), + InvalidDeliveryPromiseBasis(u8), InvalidTrace(u8), InvalidEffect(u8), NonCanonicalHolders, @@ -316,6 +415,43 @@ pub(crate) enum ShadowErrorV1 { batch_sequence: u64, reason: &'static str, }, + FrontierRecoveryWatermarkLength { + expected: usize, + actual: usize, + }, + FrontierOutputSequenceOverflow(u64), + FrontierRecoverySequence { + expected_sequence: RoundNumber, + actual_sequence: RoundNumber, + }, + FrontierRecoveryCursorAhead { + durable_sequence: RoundNumber, + actor_sequence: RoundNumber, + }, + FrontierRecoveryCursorMissing(RoundNumber), + FrontierRecoveryAnchorConflict { + consensus_round: RoundNumber, + durable: BlockReference, + actor: BlockReference, + }, + FrontierRecoveryApplicationsConflict { + consensus_round: RoundNumber, + durable: Vec, + actor: Vec, + }, + FrontierRecoveryWatermarksConflict { + consensus_round: RoundNumber, + durable: Vec, + actor: Vec, + }, + FrontierRecoveryApplicationAuthority { + application: BlockReference, + committee_size: usize, + }, + FrontierRecoverySuffixLimit { + limit: usize, + actual: usize, + }, UnrequestedRecovery(BlockReference), SlotCandidateLimit { author: AuthorityIndex, @@ -370,6 +506,67 @@ impl fmt::Display for ShadowErrorV1 { formatter, "shadow replay policy violation in WAL batch {batch_sequence}: {reason}" ), + Self::FrontierRecoveryWatermarkLength { expected, actual } => write!( + formatter, + "RBC-DAG frontier recovery cursor watermark length mismatch: expected {expected}, got {actual}" + ), + Self::FrontierOutputSequenceOverflow(sequence) => write!( + formatter, + "RBC-DAG frontier output sequence {sequence} exceeds the u32 receipt encoding" + ), + Self::FrontierRecoverySequence { + expected_sequence, + actual_sequence, + } => write!( + formatter, + "RBC-DAG frontier recovery output sequence mismatch: expected {expected_sequence}, got {actual_sequence}" + ), + Self::FrontierRecoveryCursorAhead { + durable_sequence, + actor_sequence, + } => write!( + formatter, + "RBC-DAG frontier recovery cursor sequence {durable_sequence} is ahead of actor WAL sequence {actor_sequence}" + ), + Self::FrontierRecoveryCursorMissing(sequence) => write!( + formatter, + "RBC-DAG frontier recovery cursor sequence {sequence} is missing from the actor WAL" + ), + Self::FrontierRecoveryAnchorConflict { + consensus_round, + durable, + actor, + } => write!( + formatter, + "RBC-DAG frontier recovery anchor conflict at consensus round {consensus_round}: durable {durable}, actor {actor}" + ), + Self::FrontierRecoveryApplicationsConflict { + consensus_round, + durable, + actor, + } => write!( + formatter, + "RBC-DAG frontier recovery application conflict at consensus round {consensus_round}: durable {durable:?}, actor {actor:?}" + ), + Self::FrontierRecoveryWatermarksConflict { + consensus_round, + durable, + actor, + } => write!( + formatter, + "RBC-DAG frontier recovery watermark conflict at consensus round {consensus_round}: durable {durable:?}, actor {actor:?}" + ), + Self::FrontierRecoveryApplicationAuthority { + application, + committee_size, + } => write!( + formatter, + "RBC-DAG frontier recovery application {application} is outside committee size {committee_size}" + ), + Self::FrontierRecoverySuffixLimit { limit, actual } => write!( + formatter, + "RBC-DAG frontier recovery suffix exceeds the bound {limit}: got {actual}" + ), Self::UnrequestedRecovery(reference) => { write!(formatter, "unrequested shadow recovery for {reference}") } @@ -517,17 +714,30 @@ pub(crate) struct StarfishRbcDagShadowV1 { wal: ShadowWalV1, candidates: BTreeMap, application_carriers: BTreeMap>, + /// Every authoritative delivery, including the optimistic ECHO path. delivered: BTreeSet, + /// The slower fallback values that additionally reached `Q` READYs. + certified_delivered: BTreeSet, authenticated_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, requested_recoveries: BTreeMap>, + promised_projection: PromisedProjectionModel, + pending_promised_references: BTreeSet, + pending_promised_projection_candidates: BTreeSet, + promised_projection_rejected: BTreeMap, projection: CertifiedProjectionModel, pending_projection_candidates: BTreeSet, projection_rejected: BTreeMap, projected_decisions: BTreeSet, + /// Exact derived index for the append-only decision history. Consensus + /// drive asks this on every newly decidable round; scanning all prior + /// decisions made a long run quadratic despite the slot being unique. + projected_decision_slots: BTreeSet, included_applications: BTreeSet, highest_projected_consensus_round: RoundNumber, + highest_committed_consensus_round: RoundNumber, + committed_output_count: u64, next_undecided_consensus_round: RoundNumber, next_local_consensus_round: RoundNumber, pending_projected_vertices: Vec, @@ -536,6 +746,15 @@ pub(crate) struct StarfishRbcDagShadowV1 { poisoned: bool, } +enum ShadowFrontierRecoveryPolicyV1 { + /// Comparison/mirror mode reconstructs state but never republishes + /// historical authoritative outputs. + Observational, + /// Embedded-authority mode reconciles Core's exact durable cursor and + /// republishes only the bounded, strictly newer actor-WAL suffix. + Authoritative(Option), +} + impl StarfishRbcDagShadowV1 { #[cfg(test)] pub(crate) fn open( @@ -562,8 +781,54 @@ impl StarfishRbcDagShadowV1 { context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, wal_sync_policy: ShadowWalSyncPolicyV1, + ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { + Self::open_with_frontier_recovery_policy( + path, + committee, + own_authority, + context, + authorizer, + wal_sync_policy, + ShadowFrontierRecoveryPolicyV1::Observational, + ) + } + + /// Open the embedded-authority runtime and reconcile the carrier WAL's + /// committed output against Core's exact durable cursor. The report owns + /// the strictly newer bounded suffix so startup can publish it before its + /// readiness barrier. + pub(crate) fn open_authoritative_with_wal_sync_policy( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + wal_sync_policy: ShadowWalSyncPolicyV1, + recovery_cursor: Option, + ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { + Self::open_with_frontier_recovery_policy( + path, + committee, + own_authority, + context, + authorizer, + wal_sync_policy, + ShadowFrontierRecoveryPolicyV1::Authoritative(recovery_cursor), + ) + } + + #[allow(clippy::too_many_arguments)] + fn open_with_frontier_recovery_policy( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + wal_sync_policy: ShadowWalSyncPolicyV1, + frontier_recovery_policy: ShadowFrontierRecoveryPolicyV1, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { validate_configuration(&committee, own_authority, context, &authorizer)?; + let committee_size = committee.committee().len(); let namespace = ShadowWalNamespaceV1::new(context, own_authority); let (wal, recovery) = ShadowWalV1::open_with_sync_policy(path, namespace, wal_sync_policy)?; let replayed_batches = recovery.batch_count(); @@ -571,6 +836,8 @@ impl StarfishRbcDagShadowV1 { let mut model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; model.enable_intrinsic_empty_data_availability(); + let promised_projection = + PromisedProjectionModel::from_committee_context(committee.clone()); let projection = CertifiedProjectionModel::from_committee_context(committee.clone()); let journal = WriteAheadJournalV1::new(context, own_authority); let mut core = Self { @@ -584,16 +851,24 @@ impl StarfishRbcDagShadowV1 { candidates: BTreeMap::new(), application_carriers: BTreeMap::new(), delivered: BTreeSet::new(), + certified_delivered: BTreeSet::new(), authenticated_slots: BTreeMap::new(), ordinarily_retained_slots: BTreeMap::new(), slot_candidates: BTreeMap::new(), requested_recoveries: BTreeMap::new(), + promised_projection, + pending_promised_references: BTreeSet::new(), + pending_promised_projection_candidates: BTreeSet::new(), + promised_projection_rejected: BTreeMap::new(), projection, pending_projection_candidates: BTreeSet::new(), projection_rejected: BTreeMap::new(), projected_decisions: BTreeSet::new(), + projected_decision_slots: BTreeSet::new(), included_applications: BTreeSet::new(), highest_projected_consensus_round: 0, + highest_committed_consensus_round: 0, + committed_output_count: 0, next_undecided_consensus_round: 1, next_local_consensus_round: 1, pending_projected_vertices: Vec::new(), @@ -606,11 +881,22 @@ impl StarfishRbcDagShadowV1 { let input = core.decode_batch(batch.records())?; core.apply_replayed(input, batch.records(), batch.sequence())?; } - // Historical decisions are reconstructed to validate replay, but are - // not re-emitted as fresh runtime observations after restart. + // Historical decisions are reconstructed to validate replay, but only + // an authoritative open may republish the exact Core-unacknowledged + // committed-frontier suffix. core.pending_projection_decisions.clear(); core.pending_projected_vertices.clear(); - core.pending_committed_frontiers.clear(); + let replayed_frontiers = std::mem::take(&mut core.pending_committed_frontiers); + let recovered_committed_frontiers = match frontier_recovery_policy { + ShadowFrontierRecoveryPolicyV1::Observational => Vec::new(), + ShadowFrontierRecoveryPolicyV1::Authoritative(cursor) => { + reconcile_authoritative_frontier_suffix( + replayed_frontiers, + cursor.as_ref(), + committee_size, + )? + } + }; let recovery_effects = core .requested_recoveries .iter() @@ -625,6 +911,7 @@ impl StarfishRbcDagShadowV1 { replayed_batches, discarded_tail_bytes, recovery_effects, + recovered_committed_frontiers, }, )) } @@ -637,6 +924,14 @@ impl StarfishRbcDagShadowV1 { self.model.can_create_carrier() } + /// Whether the open local carrier can name an exact admitted quorum from + /// the immediately preceding physical round. This is a read-only pacing + /// predicate: the reducer still validates the same parent set again when + /// the carrier is durably fixed. + pub(crate) fn local_parent_quorum_ready(&self) -> bool { + self.model.local_parent_set().is_ok() + } + pub(crate) fn pending_phase_backlog_len(&self) -> usize { self.model.pending_phase_backlog_len() } @@ -653,6 +948,7 @@ impl StarfishRbcDagShadowV1 { }) } + #[cfg(test)] pub(crate) fn admitted_reference( &self, authority: AuthorityIndex, @@ -661,6 +957,17 @@ impl StarfishRbcDagShadowV1 { self.model.admitted_reference(authority, round) } + /// Return the exact authenticated value retained for a slot, including a + /// future value that is inside the normal 64-round window but not yet + /// admitted by sequential clock advancement. + pub(crate) fn authenticated_reference( + &self, + authority: AuthorityIndex, + round: RoundNumber, + ) -> Option { + self.authenticated_slots.get(&(authority, round)).copied() + } + pub(crate) fn current_round_admitted_author_count(&self) -> usize { let round = self.local_carrier_round(); self.committee @@ -712,11 +1019,7 @@ impl StarfishRbcDagShadowV1 { reference: BlockReference, ) -> Result, ShadowErrorV1> { self.ensure_live()?; - if self - .model - .lifecycle(&reference) - .is_some_and(|lifecycle| lifecycle.data_available) - { + if self.projection.is_data_available(reference) { return Ok(Vec::new()); } self.apply_durable(ShadowInputV1::DataAvailable(reference)) @@ -730,9 +1033,7 @@ impl StarfishRbcDagShadowV1 { } pub(crate) fn carrier_data_available(&self, reference: BlockReference) -> bool { - self.model - .lifecycle(&reference) - .is_some_and(|lifecycle| lifecycle.data_available) + self.projection.is_data_available(reference) } pub(crate) fn wal_counts(&self) -> (u64, u64) { @@ -748,6 +1049,23 @@ impl StarfishRbcDagShadowV1 { self.model.delivered(authority, round) } + #[cfg(test)] + pub(crate) fn optimistic_promise_count(&self) -> usize { + self.slot_candidates + .values() + .flat_map(BTreeSet::iter) + .filter(|reference| { + self.model.delivery_promise_basis(reference) + == Some(DeliveryPromiseBasisV1::OptimisticEcho) + }) + .count() + } + + #[cfg(test)] + pub(crate) fn certified_delivery_count(&self) -> Result { + Ok(self.certified_delivered.len()) + } + /// Construct, authenticate, durably fix, and expose the next local /// carrier. M3 intentionally uses empty ACKs and no consensus vertex. pub(crate) fn create_local_carrier( @@ -857,6 +1175,10 @@ impl StarfishRbcDagShadowV1 { own_prev: BlockReference, allow_no_vote: bool, ) -> Option { + // `allow_no_vote` authorizes the complete C2/C3 fallback, including a + // late Vote when the scheduled leader exists. The service must bind + // that authorization to this logical consensus slot, not to a global + // physical-carrier heartbeat. let consensus_round = self.next_local_consensus_round(); let (strong_parents, leader_choice) = if consensus_round == 1 { let strong_parents = self @@ -880,7 +1202,10 @@ impl StarfishRbcDagShadowV1 { } else { let parent_round = consensus_round - 1; let mut by_author = BTreeMap::new(); - for parent in self.projection.projected_values_at_round(parent_round) { + for parent in self + .promised_projection + .projected_values_at_round(parent_round) + { by_author.entry(parent.author()).or_insert(parent); } let own_parent = *by_author.get(&self.own_authority)?; @@ -892,20 +1217,76 @@ impl StarfishRbcDagShadowV1 { return None; } let leader_author = self.committee.committee().elect_leader(parent_round); - let leader_choice = match by_author.get(&leader_author).copied() { - Some(leader) => LeaderChoiceV1::Vote { leader }, - None if allow_no_vote => LeaderChoiceV1::NoVote { - leader_author, - leader_round: parent_round, - }, - None => return None, + let leader = by_author.get(&leader_author).copied(); + let (strong_parents, leader_choice) = if consensus_round >= 3 { + let witness = match leader { + Some(leader) => match self + .promised_projection + .c1_strong_parent_witness(consensus_round, &[own_parent, leader]) + { + Ok(witness) => witness, + Err(_) => return None, + }, + None => None, + }; + match (leader, witness) { + (Some(leader), Some(witness)) => { + let witness_parents = match &witness { + C1StrongParentWitnessV1::Vote { + leader: witnessed, + parents, + } => { + debug_assert_eq!( + witnessed.consensus_round(), + consensus_round.saturating_sub(2) + ); + parents + } + C1StrongParentWitnessV1::DirectSkip { slot, parents } => { + debug_assert_eq!(slot.round, consensus_round.saturating_sub(2)); + parents + } + }; + if !witness_parents.contains(&own_parent) + || !witness_parents.contains(&leader) + { + return None; + } + (witness_parents.clone(), LeaderChoiceV1::Vote { leader }) + } + _ if !allow_no_vote => return None, + _ => self.fallback_strong_parents( + &by_author, + own_parent, + leader, + leader_author, + parent_round, + )?, + } + } else { + match leader { + Some(leader) => ( + self.minimal_strong_parent_quorum(&by_author, [own_parent, leader])?, + LeaderChoiceV1::Vote { leader }, + ), + None if allow_no_vote => self.fallback_strong_parents( + &by_author, + own_parent, + None, + leader_author, + parent_round, + )?, + None => return None, + } }; - let strong_parents = by_author.into_values().collect::>(); debug_assert!(strong_parents.contains(&own_parent)); (strong_parents, leader_choice) }; - let mut delivery_frontier = self.projection.closed_frontier(); + let mut delivery_frontier = self + .promised_projection + .joined_strong_parent_frontier(&strong_parents) + .ok()?; let own_entry = (own_prev.round != 0).then_some(own_prev); // The enclosing carrier is not clean yet, so its immediate physical // predecessor may be ahead of today's closed tip. The immutable @@ -921,10 +1302,104 @@ impl StarfishRbcDagShadowV1 { )) } - fn next_local_consensus_round(&self) -> RoundNumber { + fn fallback_strong_parents( + &self, + by_author: &BTreeMap, + own_parent: ConsensusVertexReference, + leader: Option, + leader_author: AuthorityIndex, + leader_round: RoundNumber, + ) -> Option<(Vec, LeaderChoiceV1)> { + match leader { + Some(leader) => Some(( + self.minimal_strong_parent_quorum(by_author, [own_parent, leader])?, + LeaderChoiceV1::Vote { leader }, + )), + None => Some(( + self.minimal_strong_parent_quorum(by_author, [own_parent])?, + LeaderChoiceV1::NoVote { + leader_author, + leader_round, + }, + )), + } + } + + fn minimal_strong_parent_quorum( + &self, + by_author: &BTreeMap, + required: [ConsensusVertexReference; N], + ) -> Option> { + self.promised_projection + .frontier_fresh_quorum(by_author.values().copied(), &required) + .ok() + .flatten() + } + + pub(crate) fn next_local_consensus_round(&self) -> RoundNumber { self.next_local_consensus_round } + pub(crate) fn projected_consensus_stake(&self, round: RoundNumber) -> Stake { + self.promised_projection.projected_stake_at_round(round) + } + + pub(crate) fn has_projected_consensus_quorum(&self, round: RoundNumber) -> bool { + self.projected_consensus_stake(round) >= self.committee.committee().quorum_threshold() + } + + #[cfg(test)] + pub(crate) fn inject_projected_consensus_for_test( + &mut self, + reference: ConsensusVertexReference, + strong_parents: Vec, + leader_choice: LeaderChoiceV1, + ) { + self.promised_projection.inject_projected_for_test( + reference, + strong_parents, + leader_choice, + ); + } + + #[cfg(test)] + pub(crate) fn set_next_local_consensus_round_for_test(&mut self, round: RoundNumber) { + self.next_local_consensus_round = round; + } + + pub(crate) fn projection_runtime_snapshot(&self) -> ProjectionRuntimeSnapshotV1 { + let decidable_round = self.highest_projected_consensus_round.saturating_sub(2); + let hol_reason = if self.next_undecided_consensus_round > decidable_round { + ProjectionHolReasonV1::InsufficientLookahead + } else { + let slot = self + .projection + .leader_slot(self.next_undecided_consensus_round); + match self.projection.direct_decision(slot) { + Err(_) => ProjectionHolReasonV1::DirectEvidencePending, + Ok(ProjectionDecisionV1::Undecided { .. }) => { + ProjectionHolReasonV1::AwaitingIndirectAnchor + } + Ok( + ProjectionDecisionV1::DirectCommit { .. } + | ProjectionDecisionV1::DirectSkip { .. } + | ProjectionDecisionV1::IndirectCommit { .. } + | ProjectionDecisionV1::IndirectSkip { .. }, + ) => ProjectionHolReasonV1::Ready, + } + }; + ProjectionRuntimeSnapshotV1 { + pending_candidates: self.pending_projection_candidates.len(), + highest_projected_round: self.highest_projected_consensus_round, + next_undecided_round: self.next_undecided_consensus_round, + next_undecided_projected_stake: self + .projection + .projected_stake_at_round(self.next_undecided_consensus_round), + last_committed_round: self.highest_committed_consensus_round, + hol_reason, + } + } + /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -968,6 +1443,26 @@ impl StarfishRbcDagShadowV1 { canonical_carrier_wire: &[u8], authentication_sidecar: &[u8], trusted_peer: AuthorityIndex, + ) -> Result { + self.receive_or_retain_from_peer_with_future_window( + canonical_carrier_wire, + authentication_sidecar, + trusted_peer, + EXECUTABLE_MODEL_BUFFER_WINDOW_V1, + ) + } + + /// Normal operation retains the full prototype elasticity window. During + /// exact catch-up the service narrows unsolicited ingress to the admission + /// horizon so replay responses are not starved behind duplicate future + /// work; requested exact-slot synchronization uses the normal window and + /// remains unaffected. + pub(crate) fn receive_or_retain_from_peer_with_future_window( + &mut self, + canonical_carrier_wire: &[u8], + authentication_sidecar: &[u8], + trusted_peer: AuthorityIndex, + future_window: RoundNumber, ) -> Result { self.ensure_live()?; if !self.committee.committee().known_authority(trusted_peer) { @@ -975,6 +1470,21 @@ impl StarfishRbcDagShadowV1 { } let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); + // A healthy quorum may run ahead of a temporarily descheduled peer, + // but arbitrary unsolicited future traffic must not consume MAC or + // signature verification and durable reducer capacity. Exact sync + // requests recover the receiver's current slot one round at a time. + if candidate.reference().round + > self + .model + .local_carrier_round() + .saturating_add(future_window.min(EXECUTABLE_MODEL_BUFFER_WINDOW_V1)) + { + return Ok(ShadowIngressOutcomeV1::new( + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer, + Vec::new(), + )); + } // Once a slot has a durably authenticated value, unsolicited replays // and conflicts cannot change the shadow state. Reject before public // signature/ML-DSA verification to keep this idempotence cheap. @@ -1268,7 +1778,7 @@ impl StarfishRbcDagShadowV1 { Ok(self.wal.shutdown()?) } - fn drive_certified_projection(&mut self) { + fn drive_certified_projection(&mut self) -> Result<(), ShadowErrorV1> { loop { let mut advanced = false; let candidates = self @@ -1298,10 +1808,67 @@ impl StarfishRbcDagShadowV1 { } } - self.drive_ordered_committer(self.highest_projected_consensus_round); + self.drive_ordered_committer(self.highest_projected_consensus_round) + } + + /// Advance the planning-only projection without emitting vertices, + /// decisions, anchors, or committed frontiers. A rejected promised value + /// is isolated here; certified projection continues independently. + fn drive_promised_projection(&mut self) { + loop { + let mut advanced = false; + let candidates = self + .pending_promised_projection_candidates + .iter() + .copied() + .collect::>(); + for reference in candidates { + match self.promised_projection.try_project(reference) { + Ok(_) => { + self.pending_promised_projection_candidates + .remove(&reference); + advanced = true; + } + Err(error) if projection_error_is_pending(&error) => {} + Err(error) => { + self.pending_promised_projection_candidates + .remove(&reference); + self.promised_projection_rejected.insert(reference, error); + } + } + } + if !advanced { + break; + } + } + } + + /// Resolve durable promise effects only once exact canonical content is + /// locally staged. Missing content remains an explicit replay-derived + /// pending reference until authenticated ingress or exact recovery stores + /// it; no placeholder can enter either projection plane or the delivered + /// application set. + fn activate_promised_references(&mut self) { + let available = self + .pending_promised_references + .iter() + .copied() + .filter(|reference| self.promised_projection.carrier_is_stored(*reference)) + .collect::>(); + for reference in available { + self.promised_projection + .mark_promised(reference) + .expect("a stored model promise must be valid in the promised plane"); + self.projection + .mark_delivered(reference) + .expect("a stored optimistic delivery must be valid in the certified plane"); + self.delivered.insert(reference); + self.requested_recoveries.remove(&reference); + self.pending_promised_references.remove(&reference); + } } - fn drive_ordered_committer(&mut self, highest_round: RoundNumber) { + fn drive_ordered_committer(&mut self, highest_round: RoundNumber) -> Result<(), ShadowErrorV1> { let decidable_round = highest_round.saturating_sub(2); loop { while self.next_undecided_consensus_round <= decidable_round @@ -1314,16 +1881,16 @@ impl StarfishRbcDagShadowV1 { self.next_undecided_consensus_round.saturating_add(1); } if self.next_undecided_consensus_round > decidable_round { - return; + return Ok(()); } let round = self.next_undecided_consensus_round; let slot = self.projection.leader_slot(round); let Ok(decision) = self.projection.direct_decision(slot) else { - return; + return Ok(()); }; match decision { ProjectionDecisionV1::DirectCommit { leader } => { - self.commit_projected_anchor(leader); + self.commit_projected_anchor(leader)?; self.record_projection_decision(decision); self.next_undecided_consensus_round = round.saturating_add(1); } @@ -1345,9 +1912,9 @@ impl StarfishRbcDagShadowV1 { } }); let Some(anchor) = later_anchor else { - return; + return Ok(()); }; - self.commit_projected_anchor(anchor); + self.commit_projected_anchor(anchor)?; let indirect = self .projection .indirect_decision(slot, anchor) @@ -1367,56 +1934,74 @@ impl StarfishRbcDagShadowV1 { } fn has_projection_decision(&self, slot: LeaderSlotV1) -> bool { - self.projected_decisions - .iter() - .any(|decision| projection_decision_slot(*decision) == slot) + self.projected_decision_slots.contains(&slot) } fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { if self.projected_decisions.insert(decision) { + let slot = projection_decision_slot(decision); + assert!( + self.projected_decision_slots.insert(slot), + "one logical leader slot cannot retain conflicting projection decisions" + ); self.pending_projection_decisions.push(decision); } } - fn commit_projected_anchor(&mut self, anchor: ConsensusVertexReference) { + fn commit_projected_anchor( + &mut self, + anchor: ConsensusVertexReference, + ) -> Result<(), ShadowErrorV1> { if self.projection.is_committed_anchor(anchor) { - return; + return Ok(()); } + let next_count = self + .committed_output_count + .checked_add(1) + .ok_or(ShadowErrorV1::FrontierOutputSequenceOverflow(u64::MAX))?; + let output_sequence = RoundNumber::try_from(next_count) + .map_err(|_| ShadowErrorV1::FrontierOutputSequenceOverflow(next_count))?; let frontier = self .projection - .effective_frontier(anchor) - .expect("a committed anchor must be clean and projected") - .to_vec(); - if let Err(error) = self.projection.record_committed_anchor(anchor) { - if self.projection.committed_frontier_dominates(&frontier) { - // The leader is logically committed, but a later anchor used - // for an indirect decision has already output its complete - // carrier prefix. Re-emitting it would regress the frontier. - return; - } - panic!("ordered clean anchors must be comparable exact frontiers: {error}"); - } + .record_committed_anchor(anchor) + .unwrap_or_else(|error| { + panic!("committed clean anchor frontiers must have an exact-prefix join: {error}") + }); + self.highest_committed_consensus_round = self + .highest_committed_consensus_round + .max(anchor.consensus_round()); let carriers = self .model .apply_frontier(&frontier) .expect("projection and reducer closed prefixes must agree"); - let applications = carriers - .iter() - .filter_map(|reference| { - self.candidates - .get(reference) - .and_then(|candidate| candidate.header().application_header()) - }) - .filter(|header| self.included_applications.insert(header.reference())) - .cloned() - .collect(); + let mut applications = Vec::new(); + let mut application_diagnostics = Vec::new(); + for carrier in &carriers { + let Some(application) = self + .candidates + .get(carrier) + .and_then(|candidate| candidate.header().application_header()) + .cloned() + else { + continue; + }; + if !self.included_applications.insert(application.reference()) { + continue; + } + applications.push(application); + application_diagnostics.push(CommittedApplicationDiagnosticV1::new(anchor, *carrier)); + } self.pending_committed_frontiers .push(CommittedFrontierDeltaV1 { + output_sequence, anchor, frontier, carriers, applications, + application_diagnostics, }); + self.committed_output_count = next_count; + Ok(()) } fn ensure_live(&self) -> Result<(), ShadowErrorV1> { @@ -1483,7 +2068,10 @@ impl StarfishRbcDagShadowV1 { self.poisoned = true; return Err(error.into()); } - self.record_committed_input(&input, &effects); + if let Err(error) = self.record_committed_input(&input, &effects) { + self.poisoned = true; + return Err(error); + } Ok(effects) } @@ -1509,7 +2097,10 @@ impl StarfishRbcDagShadowV1 { self.poisoned = true; return Err(ShadowErrorV1::PostModelJournal(error)); } - self.record_committed_input(&input, &effects); + if let Err(error) = self.record_committed_input(&input, &effects) { + self.poisoned = true; + return Err(error); + } Ok(effects) } @@ -1583,11 +2174,17 @@ impl StarfishRbcDagShadowV1 { Ok(()) } - fn record_committed_input(&mut self, input: &ShadowInputV1, effects: &[ModelEffect]) { + fn record_committed_input( + &mut self, + input: &ShadowInputV1, + effects: &[ModelEffect], + ) -> Result<(), ShadowErrorV1> { if let Some(candidate) = input.candidate().cloned() { let reference = candidate.reference(); let slot = carrier_slot(reference); if let Some(vertex) = candidate.header().consensus_vertex() { + self.pending_promised_projection_candidates + .insert(reference); self.pending_projection_candidates.insert(reference); if input.is_local() { self.next_local_consensus_round = self @@ -1604,12 +2201,24 @@ impl StarfishRbcDagShadowV1 { self.projection .stage_carrier(candidate.clone()) .expect("durably validated carrier must match projection committee"); - if candidate.header().transactions_commitment() == TransactionsCommitment::default() - || input.is_local() + self.promised_projection + .stage_carrier(candidate.clone()) + .expect("durably validated carrier must match promised projection committee"); + // Control carriers have no application materialization boundary, + // so the exact carrier bytes make them intrinsically available. + // An embedded application with an empty transaction commitment + // still needs its canonical header installed in Core before an + // authoritative frontier may reference it; the typed + // DataAvailable callback records that separate fact. + if candidate.header().application_header().is_none() + && candidate.header().transactions_commitment() == TransactionsCommitment::default() { self.projection .mark_data_available(reference) .expect("staged control carrier is available"); + self.promised_projection + .mark_data_available(reference) + .expect("staged control carrier is available to promised projection"); } self.candidates.insert(reference, candidate); self.slot_candidates @@ -1634,6 +2243,9 @@ impl StarfishRbcDagShadowV1 { self.projection .mark_data_available(*reference) .expect("model accepted availability only for a staged carrier"); + self.promised_projection + .mark_data_available(*reference) + .expect("model accepted availability only for a promised staged carrier"); } for effect in effects { match effect { @@ -1641,16 +2253,22 @@ impl StarfishRbcDagShadowV1 { self.requested_recoveries.insert(*target, holders.clone()); } ModelEffect::Delivered(delivered) => { + self.certified_delivered.insert(*delivered); self.delivered.insert(*delivered); self.requested_recoveries.remove(delivered); self.projection .mark_delivered(*delivered) .expect("model delivery must name a staged carrier"); } + ModelEffect::DeliveryPromised(reference) => { + self.pending_promised_references.insert(*reference); + } ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} } } - self.drive_certified_projection(); + self.activate_promised_references(); + self.drive_promised_projection(); + self.drive_certified_projection() } fn decode_batch(&self, records: &[Vec]) -> Result { @@ -1911,6 +2529,14 @@ fn journal_transition_events( context, target: *target, }, + RbcPhaseStatementV1::Vote { target } => JournalEventV1::LockVote { + context, + target: *target, + }, + RbcPhaseStatementV1::Ack { target } => JournalEventV1::LockAck { + context, + target: *target, + }, }), ModelTraceEvent::PhaseBatchEntryApplied { outer, @@ -1960,6 +2586,17 @@ fn journal_transition_events( consensus_round: *consensus_round, choice: *choice, }), + ModelTraceEvent::DeliveryPromiseLocked { target, basis } => match basis { + DeliveryPromiseBasisV1::LocalFixed + | DeliveryPromiseBasisV1::HonestAuthor + | DeliveryPromiseBasisV1::OptimisticEcho => { + Some(JournalEventV1::LockOptimisticDelivery { + context, + target: *target, + }) + } + DeliveryPromiseBasisV1::Delivered => None, + }, ModelTraceEvent::DeliveryLocked(target) => Some(JournalEventV1::LockDelivery { context, target: *target, @@ -2159,6 +2796,7 @@ fn decode_raw_record( | RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY | RECORD_LOCAL_OUTBOUND_CONTENT + | RECORD_DATA_AVAILABLE | RECORD_MODEL_TRACE | RECORD_LOCAL_OUTBOUND_SIDECAR | RECORD_LOCAL_OUTBOUND_EXPOSE => {} @@ -2180,7 +2818,10 @@ fn decode_recorded_trace( let range = match decoded.first().map(|record| record.kind) { Some(RECORD_LOCAL_OUTBOUND_CONTENT) => 1..decoded.len().saturating_sub(2), Some( - RECORD_AUTHENTICATED_INGRESS | RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY, + RECORD_AUTHENTICATED_INGRESS + | RECORD_CANDIDATE_RETENTION + | RECORD_CANDIDATE_RECOVERY + | RECORD_DATA_AVAILABLE, ) => 1..decoded.len(), _ => return Err(ShadowErrorV1::InvalidBatch("missing model input")), }; @@ -2279,6 +2920,16 @@ fn encode_trace(trace: &ModelTraceEvent) -> Result, ShadowCodecErrorV1> bytes.extend_from_slice(&consensus_round.to_be_bytes()); push_leader_choice(&mut bytes, *choice); } + ModelTraceEvent::DeliveryPromiseLocked { target, basis } => { + bytes.push(TRACE_DELIVERY_PROMISE_LOCKED); + push_reference(&mut bytes, *target); + bytes.push(match basis { + DeliveryPromiseBasisV1::LocalFixed => 0, + DeliveryPromiseBasisV1::HonestAuthor => 1, + DeliveryPromiseBasisV1::OptimisticEcho => 2, + DeliveryPromiseBasisV1::Delivered => 3, + }); + } ModelTraceEvent::DeliveryLocked(reference) => { bytes.push(TRACE_DELIVERY_LOCKED); push_reference(&mut bytes, *reference); @@ -2319,6 +2970,16 @@ fn decode_trace( consensus_round: decoder.read_u32()?, choice: decoder.read_leader_choice()?, }, + TRACE_DELIVERY_PROMISE_LOCKED => ModelTraceEvent::DeliveryPromiseLocked { + target: decoder.read_reference()?, + basis: match decoder.read_u8()? { + 0 => DeliveryPromiseBasisV1::LocalFixed, + 1 => DeliveryPromiseBasisV1::HonestAuthor, + 2 => DeliveryPromiseBasisV1::OptimisticEcho, + 3 => DeliveryPromiseBasisV1::Delivered, + other => return Err(ShadowCodecErrorV1::InvalidDeliveryPromiseBasis(other)), + }, + }, TRACE_DELIVERY_LOCKED => ModelTraceEvent::DeliveryLocked(decoder.read_reference()?), TRACE_EFFECT => ModelTraceEvent::Effect(decoder.read_effect(committee_size)?), other => return Err(ShadowCodecErrorV1::InvalidTrace(other)), @@ -2343,6 +3004,10 @@ fn push_effect(bytes: &mut Vec, effect: &ModelEffect) -> Result<(), ShadowCo bytes.push(EFFECT_DELIVERED); push_reference(bytes, *reference); } + ModelEffect::DeliveryPromised(reference) => { + bytes.push(EFFECT_DELIVERY_PROMISED); + push_reference(bytes, *reference); + } ModelEffect::PrefixAdvanced { authority, tip } => { bytes.push(EFFECT_PREFIX_ADVANCED); bytes.extend_from_slice(&authority.to_be_bytes()); @@ -2366,8 +3031,16 @@ fn push_phase(bytes: &mut Vec, statement: RbcPhaseStatementV1) { bytes.push(PHASE_READY); push_reference(bytes, target); } - } -} + RbcPhaseStatementV1::Vote { target } => { + bytes.push(PHASE_VOTE); + push_reference(bytes, target); + } + RbcPhaseStatementV1::Ack { target } => { + bytes.push(PHASE_ACK); + push_reference(bytes, target); + } + } +} fn push_consensus_reference(bytes: &mut Vec, reference: ConsensusVertexReference) { push_reference(bytes, reference.carrier()); @@ -2495,6 +3168,108 @@ fn projection_decision_slot(decision: ProjectionDecisionV1) -> LeaderSlotV1 { } } +fn reconcile_authoritative_frontier_suffix( + replayed: Vec, + cursor: Option<&RbcDagFrontierRecoveryCursorV1>, + committee_size: usize, +) -> Result, ShadowErrorV1> { + if let Some(cursor) = cursor { + if cursor.receipt.committed_rounds.len() != committee_size { + return Err(ShadowErrorV1::FrontierRecoveryWatermarkLength { + expected: committee_size, + actual: cursor.receipt.committed_rounds.len(), + }); + } + } + + let mut committed_rounds = vec![0; committee_size]; + let mut last_sequence: RoundNumber = 0; + let mut cursor_found = cursor.is_none(); + let mut suffix = Vec::new(); + for delta in replayed { + let expected_sequence = + last_sequence + .checked_add(1) + .ok_or(ShadowErrorV1::FrontierOutputSequenceOverflow( + u64::from(last_sequence) + 1, + ))?; + if delta.output_sequence != expected_sequence { + return Err(ShadowErrorV1::FrontierRecoverySequence { + expected_sequence, + actual_sequence: delta.output_sequence, + }); + } + last_sequence = delta.output_sequence; + let consensus_round = delta.anchor.consensus_round(); + + let application_references = delta + .applications + .iter() + .map(RbcCanonicalHeader::reference) + .collect::>(); + for application in &application_references { + let Some(watermark) = committed_rounds.get_mut(application.authority as usize) else { + return Err(ShadowErrorV1::FrontierRecoveryApplicationAuthority { + application: *application, + committee_size, + }); + }; + *watermark = (*watermark).max(application.round); + } + + match cursor { + Some(cursor) if delta.output_sequence < cursor.receipt.output_sequence => {} + Some(cursor) if delta.output_sequence == cursor.receipt.output_sequence => { + if delta.anchor.carrier() != cursor.receipt.carrier_anchor { + return Err(ShadowErrorV1::FrontierRecoveryAnchorConflict { + consensus_round, + durable: cursor.receipt.carrier_anchor, + actor: delta.anchor.carrier(), + }); + } + if application_references != cursor.application_references { + return Err(ShadowErrorV1::FrontierRecoveryApplicationsConflict { + consensus_round, + durable: cursor.application_references.clone(), + actor: application_references, + }); + } + if committed_rounds != cursor.receipt.committed_rounds { + return Err(ShadowErrorV1::FrontierRecoveryWatermarksConflict { + consensus_round, + durable: cursor.receipt.committed_rounds.clone(), + actor: committed_rounds.clone(), + }); + } + cursor_found = true; + } + Some(_) => suffix.push(delta), + None => suffix.push(delta), + } + } + + if let Some(cursor) = cursor { + if !cursor_found { + if cursor.receipt.output_sequence > last_sequence { + return Err(ShadowErrorV1::FrontierRecoveryCursorAhead { + durable_sequence: cursor.receipt.output_sequence, + actor_sequence: last_sequence, + }); + } + return Err(ShadowErrorV1::FrontierRecoveryCursorMissing( + cursor.receipt.output_sequence, + )); + } + } + if suffix.len() > MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 { + return Err(ShadowErrorV1::FrontierRecoverySuffixLimit { + limit: MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1, + actual: suffix.len(), + }); + } + Ok(suffix) +} + struct RawDecoder<'a> { bytes: &'a [u8], position: usize, @@ -2556,6 +3331,8 @@ impl<'a> RawDecoder<'a> { match phase { PHASE_ECHO => Ok(RbcPhaseStatementV1::Echo { target }), PHASE_READY => Ok(RbcPhaseStatementV1::Ready { target }), + PHASE_VOTE => Ok(RbcPhaseStatementV1::Vote { target }), + PHASE_ACK => Ok(RbcPhaseStatementV1::Ack { target }), other => Err(ShadowCodecErrorV1::InvalidPhase(other)), } } @@ -2602,6 +3379,7 @@ impl<'a> RawDecoder<'a> { Ok(ModelEffect::NeedCarrier { target, holders }) } EFFECT_DELIVERED => Ok(ModelEffect::Delivered(self.read_reference()?)), + EFFECT_DELIVERY_PROMISED => Ok(ModelEffect::DeliveryPromised(self.read_reference()?)), EFFECT_PREFIX_ADVANCED => Ok(ModelEffect::PrefixAdvanced { authority: self.read_u16()?, tip: self.read_reference()?, @@ -2644,7 +3422,9 @@ mod tests { MAC_TAG_SIZE, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, - starfish_rbc_dag::{RbcDagProtocolInstanceId, carrier_genesis_reference}, + starfish_rbc_dag::{ + RbcDagProtocolInstanceId, carrier_genesis_reference, journal::RbcSlotKeyV1, + }, }; const N: usize = 4; @@ -2696,8 +3476,13 @@ mod tests { self.directories[authority].path().join("shadow.wal") } - fn run_three_rounds_with_one_poisoned_recipient(&mut self) { - for round in 1..=3 { + fn run_four_phase_rounds_with_one_poisoned_recipient(&mut self) { + // INIT/ECHO is exposed in round one, the accumulated ECHOs drive + // VOTE+ACK in round two, ACK convergence drives READY in round + // three, and the Q-READY certificate is observed in round four. + // Keep these as distinct physical carrier rounds: collapsing the + // final transition would fail to exercise the runtime backlog. + for round in 1..=4 { let envelopes = self .nodes .iter_mut() @@ -2752,9 +3537,188 @@ mod tests { } #[test] - fn three_round_mac_shadow_delivers_round_one_after_poisoned_tag_is_only_staged() { + fn vote_and_ack_trace_codec_preserves_append_only_golden_tags() { + let target = BlockReference { + authority: 0x0123, + round: 0x0405_0607, + digest: BlockDigest::from([0xA5; 32]), + }; + let mut vote_golden = vec![ + TRACE_LOCAL_PHASE_LOCKED, + PHASE_VOTE, + 0x01, + 0x23, + 0x04, + 0x05, + 0x06, + 0x07, + ]; + vote_golden.extend_from_slice(&[0xA5; 32]); + let mut ack_golden = vote_golden.clone(); + ack_golden[1] = PHASE_ACK; + + let vote = ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Vote { target }); + let ack = ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Ack { target }); + assert_eq!(encode_trace(&vote).unwrap(), vote_golden); + assert_eq!(encode_trace(&ack).unwrap(), ack_golden); + assert_eq!(decode_trace(&vote_golden, N).unwrap(), vote); + assert_eq!(decode_trace(&ack_golden, N).unwrap(), ack); + + let batch_payload = encode_trace_batch(&[vote.clone(), ack.clone()]).unwrap(); + let committee = Committee::new_test(vec![1; N]); + let committee = RbcDagCommitteeContextV1::new(committee).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xD4; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ); + let raw = encode_raw_record(context, 0, RECORD_MODEL_TRACE, &batch_payload).unwrap(); + assert_eq!(&raw[..4], b"SRD5"); + let decoded = decode_raw_record(&raw, context, 0).unwrap(); + assert_eq!(decoded.kind, RECORD_MODEL_TRACE); + assert_eq!( + decode_trace_batch(&decoded.payload, N).unwrap(), + vec![vote, ack] + ); + } + + #[test] + fn vote_and_ack_local_locks_and_pending_phases_survive_shadow_reopen() { + let mut network = TestNetwork::new(); + let target = round_one_candidate(0, &network.committee, 0xD5); + let target_reference = target.reference(); + let authentication = network + .context + .authenticate_with_committee( + &target, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 0, + keys: &network.keyrings[0], + }, + ) + .unwrap(); + network.nodes[3] + .receive_authenticated_from_peer( + &target.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 0, + ) + .unwrap(); + + let outer = round_two_phase_carrier( + 1, + RbcPhaseStatementV1::Echo { + target: target_reference, + }, + &network.committee, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + network.nodes[3] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 1, + ) + .unwrap(); + + // Phase statements for a round-one target become eligible only once + // the local physical clock opens round two. Fix the local round-one + // carrier and admit one more round-one peer to expose the exact + // pending VOTE/ACK batch before restart. + network.nodes[3] + .create_local_control_heartbeat(10, true) + .unwrap(); + let peer = round_one_candidate(2, &network.committee, 0xD6); + let authentication = network + .context + .authenticate_with_committee( + &peer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 2, + keys: &network.keyrings[2], + }, + ) + .unwrap(); + network.nodes[3] + .receive_authenticated_from_peer( + &peer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 2, + ) + .unwrap(); + assert_eq!(network.nodes[3].local_carrier_round(), 2); + + let expected = [ + RbcPhaseStatementV1::Vote { + target: target_reference, + }, + RbcPhaseStatementV1::Ack { + target: target_reference, + }, + ]; + let slot = RbcSlotKeyV1::of(target_reference); + for phase in expected { + let pending = network.nodes[3].model.pending_phase_batch(); + assert!( + pending.contains(&phase), + "missing {phase:?} from pending phase batch {pending:?}" + ); + } + assert_eq!( + network.nodes[3].journal.snapshot().vote_lock(slot), + Some(target_reference) + ); + assert_eq!( + network.nodes[3].journal.snapshot().ack_lock(slot), + Some(target_reference) + ); + + let node = network.nodes.swap_remove(3); + let path = network.path(3); + node.shutdown().unwrap(); + let (reopened, report) = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 3, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[3].clone()), + ) + .unwrap(); + assert!(report.replayed_batches() >= 2); + for phase in expected { + let pending = reopened.model.pending_phase_batch(); + assert!( + pending.contains(&phase), + "reopen lost {phase:?} from pending phase batch {pending:?}" + ); + } + assert_eq!( + reopened.journal.snapshot().vote_lock(slot), + Some(target_reference) + ); + assert_eq!( + reopened.journal.snapshot().ack_lock(slot), + Some(target_reference) + ); + reopened.shutdown().unwrap(); + } + + #[test] + fn four_phase_mac_shadow_delivers_round_one_after_poisoned_tag_is_only_staged() { let mut network = TestNetwork::new(); - network.run_three_rounds_with_one_poisoned_recipient(); + network.run_four_phase_rounds_with_one_poisoned_recipient(); for node in &network.nodes { for author in 0..N { @@ -2785,7 +3749,20 @@ mod tests { let before = node.wal_counts(); let (heartbeat, effects) = node.create_local_control_heartbeat(123, true).unwrap(); - assert!(effects.is_empty()); + assert_eq!( + effects, + vec![ + ModelEffect::DeliveryPromised(heartbeat.reference()), + ModelEffect::PrefixAdvanced { + authority: 0, + tip: heartbeat.reference(), + }, + ] + ); + assert_eq!( + node.model.delivery_promise_basis(&heartbeat.reference()), + Some(DeliveryPromiseBasisV1::LocalFixed) + ); assert_eq!(node.wal_counts().0, before.0 + 1); let candidate = decode_candidate( heartbeat.canonical_carrier_wire(), @@ -2825,21 +3802,501 @@ mod tests { assert_eq!(node.admitted_reference(0, 1), Some(heartbeat.reference())); assert_eq!(node.current_round_admitted_author_count(), 1); assert_eq!(node.current_round_admitted_stake(), 1); - assert_eq!(node.pending_phase_backlog_len(), 1); + assert_eq!( + node.pending_phase_backlog_len(), + 0, + "the target author is excluded from ECHO/VOTE/ACK" + ); assert_eq!(node.buffered_authenticated_carrier_count(), 0); - let durable_counts = node.wal_counts(); - assert!(matches!( - node.create_local_control_heartbeat(124, true), - Err(ShadowErrorV1::Model(ModelError::LocalCarrierAlreadyFixed( - 1 - ))) - )); - assert_eq!(node.wal_counts(), durable_counts); + let durable_counts = node.wal_counts(); + assert!(matches!( + node.create_local_control_heartbeat(124, true), + Err(ShadowErrorV1::Model(ModelError::LocalCarrierAlreadyFixed( + 1 + ))) + )); + assert_eq!(node.wal_counts(), durable_counts); + + let mut trailing = heartbeat.canonical_carrier_wire().to_vec(); + trailing.push(0); + assert!(node.candidate_slot(&trailing).is_err()); + } + + #[test] + fn optimistic_delivery_projects_before_q_ready_for_the_same_locked_vertex() { + let mut network = TestNetwork::new(); + let target = round_one_consensus_candidate( + 1, + &network.committee, + 0x91, + TransactionsCommitment::from_bytes([0x91; 32]), + ); + let target_reference = target.reference(); + let vertex_reference = ConsensusVertexReference::new(target_reference, 1); + let authentication = network + .context + .authenticate_with_committee( + &target, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + assert!( + network.nodes[0] + .receive_authenticated_from_peer( + &target.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 1, + ) + .unwrap() + .is_empty() + ); + + // The target author's ECHO is excluded from the optimistic + // certificate. Receiving only that statement cannot promise. + let sender = 1; + { + let outer = phase_carrier( + sender, + 2, + RbcPhaseStatementV1::Echo { + target: target_reference, + }, + &network.committee, + 0xA0 + sender as u8, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: sender, + keys: &network.keyrings[sender as usize], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + sender, + ) + .unwrap(); + } + assert_eq!( + network.nodes[0] + .model + .delivery_promise_basis(&target_reference), + None + ); + assert!( + !network.nodes[0] + .promised_projection + .is_projected(vertex_reference) + ); + assert!(!network.nodes[0].projection.is_projected(vertex_reference)); + + // The receiver's local ECHO plus one other non-author ECHO reaches + // the N=4 optimistic threshold O=2. This is deliberately earlier + // than the later Q-READY delivery certificate. + let sender = 2; + let outer = phase_carrier( + sender, + 2, + RbcPhaseStatementV1::Echo { + target: target_reference, + }, + &network.committee, + 0xA2, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: sender, + keys: &network.keyrings[sender as usize], + }, + ) + .unwrap(); + let effects = network.nodes[0] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + sender, + ) + .unwrap(); + assert!(effects.contains(&ModelEffect::DeliveryPromised(target_reference))); + assert_eq!( + network.nodes[0] + .model + .delivery_promise_basis(&target_reference), + Some(DeliveryPromiseBasisV1::OptimisticEcho) + ); + assert_eq!(network.nodes[0].promised_projection.promised_tip(1), None); + assert!( + !network.nodes[0] + .promised_projection + .is_projected(vertex_reference) + ); + assert!(!network.nodes[0].carrier_data_available(target_reference)); + + network.nodes[0] + .mark_carrier_data_available(target_reference) + .unwrap(); + assert!( + network.nodes[0] + .promised_projection + .is_projected(vertex_reference) + ); + assert_eq!( + network.nodes[0].promised_projection.promised_tip(1), + Some(target_reference) + ); + assert!(network.nodes[0].projection.is_projected(vertex_reference)); + assert_eq!( + network.nodes[0].projection.closed_tip(1), + Some(target_reference) + ); + assert_eq!( + network.nodes[0].drain_projected_vertices(), + vec![vertex_reference] + ); + assert!(network.nodes[0].drain_projection_decisions().is_empty()); + assert!(network.nodes[0].drain_committed_frontiers().is_empty()); + + // Reopening from typed inputs and trace effects reconstructs the + // authoritative optimistic delivery and the identical planning view. + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let mut node = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap() + .0; + assert!(node.promised_projection.is_projected(vertex_reference)); + assert!(node.projection.is_projected(vertex_reference)); + assert!( + !node + .model + .lifecycle(&target_reference) + .unwrap() + .certified_delivered + ); + assert!(node.drain_committed_frontiers().is_empty()); + + // Two external READYs first trigger the local READY and then produce + // the final Q-READY delivery. Certified projection catches up to the + // exact immutable vertex and effective frontier planned earlier. + for sender in [1, 2] { + let outer = phase_carrier( + sender, + 3, + RbcPhaseStatementV1::Ready { + target: target_reference, + }, + &network.committee, + 0xB0 + sender as u8, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: sender, + keys: &network.keyrings[sender as usize], + }, + ) + .unwrap(); + node.receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + sender, + ) + .unwrap(); + } + assert!(node.model.lifecycle(&target_reference).unwrap().delivered); + assert!( + node.model + .lifecycle(&target_reference) + .unwrap() + .certified_delivered + ); + assert!(node.projection.is_projected(vertex_reference)); + assert_eq!( + node.promised_projection.projected_vertex(vertex_reference), + node.projection.projected_vertex(vertex_reference) + ); + assert_eq!( + node.promised_projection + .effective_frontier(vertex_reference), + node.projection.effective_frontier(vertex_reference) + ); + for projected in node.drain_projected_vertices() { + assert!(node.projection.is_projected(projected)); + } + for delta in node.drain_committed_frontiers() { + assert!(node.projection.is_projected(delta.anchor)); + assert!(delta.carriers.iter().all(|reference| { + node.model.lifecycle(reference).is_some_and(|lifecycle| { + lifecycle.delivered && lifecycle.data_available && lifecycle.prefix_closed + }) + })); + } + } + + #[test] + fn promise_for_missing_content_waits_for_exact_recovery_without_panicking() { + let mut network = TestNetwork::new(); + let target = round_one_consensus_candidate( + 1, + &network.committee, + 0x92, + TransactionsCommitment::default(), + ); + let target_reference = target.reference(); + let vertex_reference = ConsensusVertexReference::new(target_reference, 1); + let unrelated = round_one_candidate(2, &network.committee, 0x93); + + // This directly exercises the durable-adapter boundary represented by + // an all-author-ECHO promise whose exact carrier bytes have not yet + // arrived. The effect is retained, not applied to a placeholder. + network.nodes[0] + .record_committed_input( + &ShadowInputV1::CandidateRetention(unrelated), + &[ModelEffect::DeliveryPromised(target_reference)], + ) + .unwrap(); + assert!( + network.nodes[0] + .pending_promised_references + .contains(&target_reference) + ); + assert!( + !network.nodes[0] + .promised_projection + .is_projected(vertex_reference) + ); + + network.nodes[0] + .record_committed_input(&ShadowInputV1::CandidateRecovery(target), &[]) + .unwrap(); + assert!(network.nodes[0].pending_promised_references.is_empty()); + assert!( + network.nodes[0] + .promised_projection + .is_projected(vertex_reference) + ); + assert!(network.nodes[0].projection.is_projected(vertex_reference)); + } + + #[test] + fn missing_content_echo_evidence_reopens_then_promises_on_exact_recovery() { + let mut network = TestNetwork::new(); + let target = round_one_consensus_candidate( + 3, + &network.committee, + 0x94, + TransactionsCommitment::default(), + ); + let target_reference = target.reference(); + let vertex_reference = ConsensusVertexReference::new(target_reference, 1); + + // Three remote ECHOs reach Q while exact content is missing. The + // receiver cannot count its own ECHO without first authenticating the + // carrier, so the reducer requests recovery but emits no promise. + for sender in [1, 2, 3] { + let outer = phase_carrier( + sender, + 2, + RbcPhaseStatementV1::Echo { + target: target_reference, + }, + &network.committee, + 0xC0 + sender as u8, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: sender, + keys: &network.keyrings[sender as usize], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + sender, + ) + .unwrap(); + } + assert!( + network.nodes[0] + .retained_candidate_wire(target_reference) + .is_none() + ); + assert_eq!( + network.nodes[0] + .model + .delivery_promise_basis(&target_reference), + None + ); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let (mut node, report) = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap(); + assert!(report.recovery_effects().iter().any(|effect| { + matches!(effect, ModelEffect::NeedCarrier { target, .. } if *target == target_reference) + })); + assert_eq!(node.model.delivery_promise_basis(&target_reference), None); + + let recovery_effects = node + .recover_candidate_for(target_reference, &target.canonical_wire_bytes().unwrap()) + .unwrap(); + assert!(node.retained_candidate_wire(target_reference).is_some()); + assert!( + recovery_effects.contains(&ModelEffect::DeliveryPromised(target_reference)), + "the persisted ECHO certificate must activate once exact content arrives" + ); + assert_eq!( + node.model.delivery_promise_basis(&target_reference), + Some(DeliveryPromiseBasisV1::OptimisticEcho) + ); + assert!(node.promised_projection.is_projected(vertex_reference)); + + // Later receiver-authenticated ingress may authorize the local ECHO, + // but it must not duplicate the already durable promise. + let authentication = network + .context + .authenticate_with_committee( + &target, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 3, + keys: &network.keyrings[3], + }, + ) + .unwrap(); + let effects = node + .receive_authenticated_from_peer( + &target.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + 3, + ) + .unwrap(); + assert!(!effects.contains(&ModelEffect::DeliveryPromised(target_reference))); + assert!(node.promised_projection.is_projected(vertex_reference)); + assert!(node.projection.is_projected(vertex_reference)); + assert!( + !node + .model + .lifecycle(&target_reference) + .unwrap() + .certified_delivered + ); + } + + #[test] + fn c1_builder_waits_for_the_witness_and_avoids_the_mixed_slow_tail() { + let committee = Committee::new_test(vec![1; 7]); + let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xC1; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, + ); + let keyrings = mac_keyrings_for_test(7); + let directory = tempfile::tempdir().unwrap(); + let mut node = StarfishRbcDagShadowV1::open( + directory.path().join("c1-builder.wal"), + committee.clone(), + 0, + context, + ShadowAuthorizerV1::MacVector(keyrings[0].clone()), + ) + .unwrap() + .0; + let reference = |authority, consensus_round, marker| { + ConsensusVertexReference::new( + BlockReference { + authority, + round: consensus_round + 100, + digest: BlockDigest::from([marker; 32]), + }, + consensus_round, + ) + }; + + let target = reference(committee.committee().elect_leader(2), 2, 0x20); + node.promised_projection.inject_projected_for_test( + target, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: committee.committee().elect_leader(1), + leader_round: 1, + }, + ); + for author in [0, 1, 2, 3, 6] { + let projected = reference(author, 3, 0x30 + author as u8); + let choice = if author == 6 { + LeaderChoiceV1::NoVote { + leader_author: target.author(), + leader_round: target.consensus_round(), + } + } else { + LeaderChoiceV1::Vote { leader: target } + }; + node.promised_projection + .inject_projected_for_test(projected, vec![target], choice); + } + node.next_local_consensus_round = 4; + assert!( + node.build_local_consensus_vertex(carrier_genesis_reference(0), false) + .is_none(), + "a mixed first quorum must not fix a non-certifying C1 vertex" + ); - let mut trailing = heartbeat.canonical_carrier_wire().to_vec(); - trailing.push(0); - assert!(node.candidate_slot(&trailing).is_err()); + let final_voter = reference(4, 3, 0x34); + node.promised_projection.inject_projected_for_test( + final_voter, + vec![target], + LeaderChoiceV1::Vote { leader: target }, + ); + let vertex = node + .build_local_consensus_vertex(carrier_genesis_reference(0), false) + .expect("the exact fifth vote completes C1"); + let authors = vertex + .strong_parents() + .iter() + .map(|parent| parent.author()) + .collect::>(); + assert_eq!(authors, vec![0, 1, 2, 3, 4]); + assert_eq!(vertex.strong_parents().len(), 5); + assert!(!authors.contains(&6)); } #[test] @@ -2926,6 +4383,152 @@ mod tests { assert!(restarted.journal.snapshot().leader_choice(1).is_some()); } + #[test] + fn empty_application_waits_for_explicit_core_materialization() { + let mut network = TestNetwork::new(); + let application_header = RbcCanonicalHeader::try_new( + 0, + 1, + network + .committee + .committee() + .authorities() + .map(carrier_genesis_reference) + .collect(), + Vec::new(), + 899, + TransactionsCommitment::default(), + ) + .unwrap(); + let reference = network.nodes[0] + .create_local_application_carrier(application_header, 999, true) + .unwrap() + .0 + .reference(); + + assert!( + network.nodes[0] + .model + .lifecycle(&reference) + .unwrap() + .data_available, + "the empty payload is intrinsically available to the RBC reducer" + ); + assert!( + !network.nodes[0].projection.is_data_available(reference), + "authoritative projection must still wait for the Core header" + ); + + network.nodes[0] + .mark_carrier_data_available(reference) + .unwrap(); + assert!(network.nodes[0].projection.is_data_available(reference)); + assert!( + network.nodes[0] + .promised_projection + .is_data_available(reference) + ); + } + + #[test] + fn application_data_availability_record_reopens_from_the_wal() { + let mut network = TestNetwork::new(); + let commitment = TransactionsCommitment::from_bytes([0xDA; 32]); + let application_header = RbcCanonicalHeader::try_new( + 0, + 1, + network + .committee + .committee() + .authorities() + .map(carrier_genesis_reference) + .collect(), + Vec::new(), + 900, + commitment, + ) + .unwrap(); + let reference = network.nodes[0] + .create_local_application_carrier(application_header, 1_000, true) + .unwrap() + .0 + .reference(); + assert!(!network.nodes[0].carrier_data_available(reference)); + assert_eq!(network.nodes[0].projection.closed_tip(0), None); + + for sender in [1, 2] { + let outer = round_two_phase_carrier( + sender, + RbcPhaseStatementV1::Ready { target: reference }, + &network.committee, + ); + let authentication = network + .context + .authenticate_with_committee( + &outer, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: sender, + keys: &network.keyrings[sender as usize], + }, + ) + .unwrap(); + network.nodes[0] + .receive_authenticated_from_peer( + &outer.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + sender, + ) + .unwrap(); + } + + let vertex = ConsensusVertexReference::new(reference, 1); + let lifecycle = network.nodes[0].model.lifecycle(&reference).unwrap(); + assert!(lifecycle.delivered); + assert!(!lifecycle.prefix_closed); + assert_eq!(network.nodes[0].projection.closed_tip(0), None); + assert!(matches!( + network.nodes[0].projection.try_project(reference), + Err(CertifiedProjectionError::CarrierDataUnavailable(actual)) if actual == reference + )); + + let effects = network.nodes[0] + .mark_carrier_data_available(reference) + .unwrap(); + assert!(effects.iter().any( + |effect| matches!(effect, ModelEffect::PrefixAdvanced { tip, .. } if *tip == reference) + )); + assert!(network.nodes[0].carrier_data_available(reference)); + assert!( + network.nodes[0] + .model + .lifecycle(&reference) + .unwrap() + .prefix_closed + ); + assert_eq!(network.nodes[0].projection.closed_tip(0), Some(reference)); + assert!(network.nodes[0].projection.is_projected(vertex)); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let (restarted, report) = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap(); + + assert_eq!(report.replayed_batches(), 4); + assert!(restarted.carrier_data_available(reference)); + assert!(restarted.model.lifecycle(&reference).unwrap().prefix_closed); + assert_eq!(restarted.projection.closed_tip(0), Some(reference)); + assert!(restarted.projection.is_projected(vertex)); + assert!(restarted.retained_candidate_wire(reference).is_some()); + } + #[test] fn authenticated_replays_and_slot_conflicts_do_not_grow_durable_state() { let mut network = TestNetwork::new(); @@ -3032,18 +4635,18 @@ mod tests { } #[test] - fn rejected_far_future_ingress_does_not_poison_the_durable_actor() { + fn ignored_far_future_ingress_skips_authentication_and_durable_state() { let mut network = TestNetwork::new(); let author = 1; let previous = |authority: AuthorityIndex| BlockReference { authority, - round: 5, + round: 65, digest: BlockDigest::from([0x90 + authority as u8; 32]), }; let candidate = CandidateCarrierV1::try_new_with_committee( CarrierHeaderV1Args { author, - carrier_round: 6, + carrier_round: 66, own_prev: previous(author), weak_parents: [0, 2].into_iter().map(previous).collect(), transactions_commitment: TransactionsCommitment::default(), @@ -3056,32 +4659,20 @@ mod tests { &network.committee, ) .unwrap(); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: author, - keys: &network.keyrings[author as usize], - }, - ) - .unwrap(); - - assert!(matches!( - network.nodes[0].receive_or_retain_from_peer( + let outcome = network.nodes[0] + .receive_or_retain_from_peer( &candidate.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), + // The far-future prefilter must run before parsing or + // verifying the authentication sidecar. + b"not-an-authentication-sidecar", author, - ), - Err(ShadowErrorV1::Model( - ModelError::FutureCarrierOutsideBuffer { - current: 1, - maximum: 5, - actual: 6, - } - )) - )); + ) + .unwrap(); + assert_eq!( + outcome.disposition(), + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer + ); + assert!(outcome.effects().is_empty()); assert_eq!(network.nodes[0].wal_counts(), (0, 0)); network.nodes[0] .create_local_control_heartbeat(2, true) @@ -3235,12 +4826,12 @@ mod tests { #[test] fn wal_restart_past_round_one_discards_torn_tail_and_retransmits_exact_bytes() { let mut network = TestNetwork::new(); - network.run_three_rounds_with_one_poisoned_recipient(); + network.run_four_phase_rounds_with_one_poisoned_recipient(); let node = network.nodes.swap_remove(0); let path = network.path(0); let before = node.retransmissions(); - assert_eq!(before.len(), 3); + assert_eq!(before.len(), 4); let retained = node.retained_candidate_wire(before[0].reference()).unwrap(); assert_eq!(retained, before[0].canonical_carrier_wire()); node.shutdown().unwrap(); @@ -3278,7 +4869,7 @@ mod tests { .unwrap(); assert_eq!(report.discarded_tail_bytes(), torn.len() as u64); assert!(report.replayed_batches() > 3); - assert_eq!(restarted.local_carrier_round(), 4); + assert_eq!(restarted.local_carrier_round(), 5); assert_eq!(restarted.retransmissions(), before); assert!(report.recovery_effects().is_empty()); for author in 0..N { @@ -3431,7 +5022,7 @@ mod tests { #[test] fn direct_shadow_comparison_reports_match_mismatch_and_ambiguity_without_references() { let mut network = TestNetwork::new(); - network.run_three_rounds_with_one_poisoned_recipient(); + network.run_four_phase_rounds_with_one_poisoned_recipient(); let node = &network.nodes[0]; let direct = node.delivered_identities().unwrap(); assert_eq!( @@ -3553,6 +5144,91 @@ mod tests { .unwrap() } + fn round_one_consensus_candidate( + author: AuthorityIndex, + committee: &RbcDagCommitteeContextV1, + marker: u8, + transactions_commitment: TransactionsCommitment, + ) -> CandidateCarrierV1 { + let weak_parents = committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(carrier_genesis_reference) + .collect(); + let strong_parents = committee + .committee() + .authorities() + .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) + .collect(); + let leader_author = committee.committee().elect_leader(0); + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 1, + own_prev: carrier_genesis_reference(author), + weak_parents, + transactions_commitment, + application_header: None, + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: Some(ConsensusVertexV1::new( + 1, + strong_parents, + vec![None; committee.committee().len()], + LeaderChoiceV1::Vote { + leader: ConsensusVertexReference::new( + carrier_genesis_reference(leader_author), + 0, + ), + }, + )), + creation_time_ns: u64::from(marker), + }, + committee, + ) + .unwrap() + } + + fn phase_carrier( + author: AuthorityIndex, + round: RoundNumber, + statement: RbcPhaseStatementV1, + committee: &RbcDagCommitteeContextV1, + marker: u8, + ) -> CandidateCarrierV1 { + assert!(round > 1); + let previous = |authority: AuthorityIndex| BlockReference { + authority, + round: round - 1, + digest: BlockDigest::from([marker.wrapping_add(authority as u8); 32]), + }; + let weak_parents = committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(previous) + .collect(); + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: round, + own_prev: previous(author), + weak_parents, + transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), + application_header: None, + data_acknowledgments: Vec::new(), + phase_batch: vec![statement], + consensus_vertex: None, + creation_time_ns: u64::from(round), + }, + committee, + ) + .unwrap() + } + fn round_two_phase_carrier( author: AuthorityIndex, statement: RbcPhaseStatementV1, @@ -3589,4 +5265,173 @@ mod tests { ) .unwrap() } + + fn recovery_application( + authority: AuthorityIndex, + round: RoundNumber, + marker: u8, + ) -> RbcCanonicalHeader { + RbcCanonicalHeader::try_new( + authority, + round, + (0..N as AuthorityIndex) + .map(carrier_genesis_reference) + .collect(), + Vec::new(), + u64::from(marker), + TransactionsCommitment::from_bytes([marker; 32]), + ) + .unwrap() + } + + fn recovery_delta( + output_sequence: RoundNumber, + consensus_round: RoundNumber, + applications: Vec, + ) -> CommittedFrontierDeltaV1 { + let carrier = BlockReference::new_test( + (consensus_round as usize % N) as AuthorityIndex, + consensus_round.saturating_add(100), + ); + let application_diagnostics = applications + .iter() + .map(|_| CommittedApplicationDiagnosticV1 { + physical_carrier_round_delta: 0, + }) + .collect(); + CommittedFrontierDeltaV1 { + output_sequence, + anchor: ConsensusVertexReference::new(carrier, consensus_round), + frontier: vec![None; N], + carriers: Vec::new(), + applications, + application_diagnostics, + } + } + + fn recovery_cursor( + delta: &CommittedFrontierDeltaV1, + committed_rounds: Vec, + ) -> RbcDagFrontierRecoveryCursorV1 { + RbcDagFrontierRecoveryCursorV1 { + receipt: RbcDagFrontierReceipt { + carrier_anchor: delta.anchor.carrier(), + output_sequence: delta.output_sequence, + committed_rounds, + }, + application_references: delta + .applications + .iter() + .map(RbcCanonicalHeader::reference) + .collect(), + } + } + + #[test] + fn authoritative_frontier_recovery_replays_only_the_exact_newer_suffix() { + let first_application = recovery_application(1, 5, 0xA1); + let last_application = recovery_application(2, 7, 0xA2); + // Logical anchor rounds may regress while the output sequence remains + // contiguous and monotone. + let first = recovery_delta(1, 8, vec![first_application]); + let control_only = recovery_delta(2, 3, Vec::new()); + let last = recovery_delta(3, 7, vec![last_application]); + let history = vec![first.clone(), control_only.clone(), last.clone()]; + + assert_eq!( + reconcile_authoritative_frontier_suffix(history.clone(), None, N).unwrap(), + history + ); + + let after_first = reconcile_authoritative_frontier_suffix( + history.clone(), + Some(&recovery_cursor(&first, vec![0, 5, 0, 0])), + N, + ) + .unwrap(); + assert_eq!(after_first, vec![control_only.clone(), last.clone()]); + + let after_control = reconcile_authoritative_frontier_suffix( + history, + Some(&recovery_cursor(&control_only, vec![0, 5, 0, 0])), + N, + ) + .unwrap(); + assert_eq!(after_control, vec![last]); + } + + #[test] + fn authoritative_frontier_recovery_rejects_unreconciled_cursors() { + let application = recovery_application(1, 5, 0xB1); + let first = recovery_delta(1, 8, vec![application]); + let last = recovery_delta(2, 3, Vec::new()); + let history = vec![first.clone(), last.clone()]; + + let mut conflict = recovery_cursor(&first, vec![0, 5, 0, 0]); + conflict.receipt.carrier_anchor = BlockReference::new_test(0, 999); + assert!(matches!( + reconcile_authoritative_frontier_suffix(history.clone(), Some(&conflict), N), + Err(ShadowErrorV1::FrontierRecoveryAnchorConflict { .. }) + )); + + let mut conflict = recovery_cursor(&first, vec![0, 5, 0, 0]); + conflict.application_references.clear(); + assert!(matches!( + reconcile_authoritative_frontier_suffix(history.clone(), Some(&conflict), N), + Err(ShadowErrorV1::FrontierRecoveryApplicationsConflict { .. }) + )); + + let conflict = recovery_cursor(&first, vec![0, 4, 0, 0]); + assert!(matches!( + reconcile_authoritative_frontier_suffix(history.clone(), Some(&conflict), N), + Err(ShadowErrorV1::FrontierRecoveryWatermarksConflict { .. }) + )); + + let missing = RbcDagFrontierRecoveryCursorV1 { + receipt: RbcDagFrontierReceipt { + carrier_anchor: BlockReference::new_test(2, 103), + output_sequence: 3, + committed_rounds: vec![0, 5, 0, 0], + }, + application_references: Vec::new(), + }; + assert!(matches!( + reconcile_authoritative_frontier_suffix(history.clone(), Some(&missing), N), + Err(ShadowErrorV1::FrontierRecoveryCursorAhead { + durable_sequence: 3, + actor_sequence: 2 + }) + )); + + let ahead = RbcDagFrontierRecoveryCursorV1 { + receipt: RbcDagFrontierReceipt { + carrier_anchor: BlockReference::new_test(0, 104), + output_sequence: 4, + committed_rounds: vec![0, 5, 0, 0], + }, + application_references: Vec::new(), + }; + assert!(matches!( + reconcile_authoritative_frontier_suffix(history, Some(&ahead), N), + Err(ShadowErrorV1::FrontierRecoveryCursorAhead { + durable_sequence: 4, + actor_sequence: 2 + }) + )); + } + + #[test] + fn authoritative_frontier_recovery_suffix_is_bounded() { + let history = (1..=MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 + 1) + .map(|sequence| { + recovery_delta(sequence as RoundNumber, sequence as RoundNumber, Vec::new()) + }) + .collect(); + assert!(matches!( + reconcile_authoritative_frontier_suffix(history, None, N), + Err(ShadowErrorV1::FrontierRecoverySuffixLimit { limit, actual }) + if limit == MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 + && actual == MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 + 1 + )); + } } diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 7aff59da..d4504394 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -20,31 +20,46 @@ use parking_lot::Mutex; use tokio::{ sync::{ mpsc::{self, error::TrySendError}, - oneshot, + oneshot, watch, }, task::JoinHandle, }; use crate::{ - crypto::{MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE}, + crypto::{ + MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE, + TransactionsCommitment, + }, + metrics::{ + Metrics, RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, + }, network::{ - NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse, - RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, + NetworkMessage, RbcDagApplicationPayloadResponse, RbcDagShadowCarrier, + RbcDagShadowCarrierResponse, RbcDagShadowCarrierSyncRequest, + RbcDagShadowCarrierSyncResponse, }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ - ConsensusVertexReference, MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, - RbcDagContextV1, - model::{ModelEffect, ModelError}, + CandidateCarrierV1, ConsensusVertexReference, MAX_CARRIER_CONTENT_SIZE_V1, + RbcDagCommitteeContextV1, RbcDagContextV1, + model::{ + EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, + ModelError, + }, projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ - CommittedFrontierDeltaV1, ShadowAuthorizerV1, ShadowDeliveryComparisonV1, - ShadowDeliveryIdentityV1, ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, - ShadowOpenReportV1, ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, + CommittedFrontierDeltaV1, RbcDagFrontierRecoveryCursorV1, ShadowAuthorizerV1, + ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, ShadowDeliverySlotV1, ShadowErrorV1, + ShadowIngressDispositionV1, ShadowOpenReportV1, ShadowOutboundEnvelopeV1, + StarfishRbcDagShadowV1, + }, + types::{ + AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs, + TransactionData, }, - types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, }; // A mirror run must absorb one complete committee fan-in plus a small reserve; @@ -65,7 +80,58 @@ const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); const SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1: Duration = Duration::from_millis(100); +// At most sixteen distinct exact slots may bypass the duplicate retry interval +// per requester. Even at the four-MiB carrier ceiling this bounds one +// interval's replay exposure to 64 MiB; ordinary network backpressure remains +// the second bound. Production should replace this prototype credit with a +// configured byte-rate budget. +const SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1: usize = 16; +/// Exact repair is an independently bounded priority lane. The same cap is +/// used for outstanding request slots and for responses coalesced outside the +/// ordinary actor FIFO. +const SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1: usize = 64; const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_millis(500); +/// This prototype applies the existing four-MiB canonical-content/default +/// block ceiling as a conservative serialized-payload preflight. The network +/// frame ceiling is larger; a dedicated configurable payload limit remains a +/// deployment-hardening boundary. Keeping only a bounded recent window stops +/// unsolicited sidecars from turning the actor into an unbounded cache. +const SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1: usize = SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1; +const SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1: usize = MAX_CARRIER_CONTENT_SIZE_V1; +const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); +/// Bound healthy physical-carrier production independently of actor/network +/// scheduling. The 600-ms Starfish pacemaker therefore permits at most about +/// 33 normal carriers/s, while validity-backed repair retains its separately +/// bounded burst lane. +const SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1: u32 = 20; +const SHADOW_NORMAL_CARRIER_MIN_SPACING_V1: Duration = Duration::from_millis(1); + +type CarrierSyncSlotV1 = (RoundNumber, AuthorityIndex); +type DesiredCarrierSyncResponseV1 = (AuthorityIndex, RbcDagShadowCarrierSyncResponse); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct NormalCarrierDeadlineV1 { + generation: u64, + deadline: Instant, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ConsensusTimeoutDeadlineV1 { + generation: u64, + slot: RoundNumber, + deadline: Instant, +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct CarrierSyncInspectionV1 { + open_round: RoundNumber, + target: Option, + outstanding: usize, + desired_responses: usize, + max_outstanding: usize, + max_desired_responses: usize, +} /// Runtime role of the persisted carrier actor. /// @@ -100,29 +166,138 @@ impl ShadowServiceModeV1 { } } -#[derive(Clone, Debug, Eq, PartialEq)] +/// Per-logical-slot Starfish creation pacemaker. +/// +/// The physical heartbeat runs on a fixed grid, so its tick time cannot also +/// be the C2 origin: a logical slot may have opened only an instant before the +/// tick. C2 is armed once A1 is locally true for this exact slot. C3 remains +/// an immediate, independently sufficient catch-up condition. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ConsensusPacemakerV1 { + slot: RoundNumber, + c2_armed_at: Option, + c2_timed_out: bool, +} + +impl ConsensusPacemakerV1 { + fn new(slot: RoundNumber) -> Self { + Self { + slot, + c2_armed_at: None, + c2_timed_out: false, + } + } + + fn fallback_allowed( + &mut self, + slot: RoundNumber, + a1_ready: bool, + c3_ready: bool, + leader_timeout: Duration, + now: Instant, + ) -> bool { + if self.slot != slot { + self.slot = slot; + self.c2_armed_at = None; + self.c2_timed_out = false; + } + if a1_ready { + self.c2_armed_at.get_or_insert(now); + } else { + // Eligible projection is monotonic, so this is principally a + // fail-closed guard against arming from the wrong logical slot. + self.c2_armed_at = None; + self.c2_timed_out = false; + } + c3_ready + || self.c2_timed_out + || self + .c2_armed_at + .is_some_and(|armed| now.saturating_duration_since(armed) >= leader_timeout) + } + + fn observe_timeout(&mut self, slot: RoundNumber) -> bool { + if self.slot != slot || self.c2_armed_at.is_none() { + return false; + } + self.c2_timed_out = true; + true + } +} + +#[derive(Clone, Debug)] struct ShadowLocalCarrierV1 { author: AuthorityIndex, round: RoundNumber, - transactions_commitment: crate::crypto::TransactionsCommitment, + transactions_commitment: TransactionsCommitment, creation_time_ns: TimestampNs, application_header: RbcCanonicalHeader, + application_payload: Option>, + /// True only when the live core submitted this header and is waiting for + /// an `ApplicationAssigned` flow-control acknowledgment. Recovered direct + /// history may be assigned during WAL reconciliation without a live + /// producer gate to release. + acknowledge_assignment: bool, } impl ShadowLocalCarrierV1 { + #[cfg(test)] fn from_direct_header(header: &RbcCanonicalHeader) -> Self { + Self::from_direct_header_with_payload(header, None) + } + + fn from_direct_header_with_payload( + header: &RbcCanonicalHeader, + application_payload: Option>, + ) -> Self { + Self::from_direct_header_with_ack(header, application_payload, true) + } + + fn from_recovered_direct_header(header: &RbcCanonicalHeader) -> Self { + Self::from_direct_header_with_ack(header, None, false) + } + + fn from_direct_header_with_ack( + header: &RbcCanonicalHeader, + application_payload: Option>, + acknowledge_assignment: bool, + ) -> Self { Self { author: header.reference().authority, round: header.reference().round, transactions_commitment: header.transactions_commitment(), creation_time_ns: header.meta_creation_time_ns(), application_header: header.clone(), + application_payload, + acknowledge_assignment, } } + + fn same_application(&self, other: &Self) -> bool { + self.author == other.author + && self.round == other.round + && self.transactions_commitment == other.transactions_commitment + && self.creation_time_ns == other.creation_time_ns + && self.application_header == other.application_header + } +} + +impl PartialEq for ShadowLocalCarrierV1 { + fn eq(&self, other: &Self) -> bool { + self.same_application(other) + && application_payloads_equal( + self.application_payload.as_deref(), + other.application_payload.as_deref(), + ) + && self.acknowledge_assignment == other.acknowledge_assignment + } } +impl Eq for ShadowLocalCarrierV1 {} + enum ShadowServiceMessageV1 { - LocalCarrier(ShadowLocalCarrierV1), + ActivateClock(oneshot::Sender<()>), + LocalApplicationsChanged, Carrier { peer: AuthorityIndex, envelope: RbcDagShadowCarrier, @@ -139,15 +314,32 @@ enum ShadowServiceMessageV1 { peer: AuthorityIndex, request: RbcDagShadowCarrierSyncRequest, }, - CarrierSyncResponse { + CarrierSyncResponsesChanged, + ApplicationPayloadRequest { peer: AuthorityIndex, - response: RbcDagShadowCarrierSyncResponse, + application: BlockReference, + }, + ApplicationPayloadResponse { + peer: AuthorityIndex, + response: RbcDagApplicationPayloadResponse, }, + VerifiedApplicationPayloadsChanged, DirectDeliveriesChanged, TopologyChanged, RetryRecovery, HeartbeatTick, + NormalCarrierDeadline { + generation: u64, + }, + ConsensusTimeoutDeadline { + generation: u64, + slot: RoundNumber, + }, DataAvailabilityChanged, + #[cfg(test)] + InspectRbcProgress(oneshot::Sender<(usize, usize)>), + #[cfg(test)] + InspectCarrierSync(oneshot::Sender), Shutdown(oneshot::Sender>), } @@ -157,15 +349,25 @@ pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { max_sidecar_size: usize, own_authority: AuthorityIndex, committee_size: usize, + #[cfg(test)] input_capacity: usize, mode: ShadowServiceModeV1, desired_topology: Arc>>, + desired_local_applications: Arc>>, + /// Exact `(round, author)` responses outside the ordinary actor FIFO. + /// Catch-up must not wait behind the proactive future carriers it is + /// intended to overtake. + desired_carrier_sync_responses: + Arc>>, + desired_verified_application_payloads: + Arc>>>, desired_direct_deliveries: Arc>>, desired_available_applications: Arc>>, invalidated_by_overload: Arc>>, } impl StarfishRbcDagShadowServiceHandleV1 { + #[cfg(test)] fn send(&self, message: ShadowServiceMessageV1) -> Result<(), ShadowServiceErrorV1> { let kind = message.kind(); if let Some(reason) = *self.invalidated_by_overload.lock() { @@ -183,15 +385,72 @@ impl StarfishRbcDagShadowServiceHandleV1 { }) } + async fn send_reliably( + &self, + message: ShadowServiceMessageV1, + ) -> Result<(), ShadowServiceErrorV1> { + if let Some(reason) = *self.invalidated_by_overload.lock() { + return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); + } + self.sender + .send(message) + .await + .map_err(|_| ShadowServiceErrorV1::Stopped) + } + pub(crate) fn local_header( &self, header: &RbcCanonicalHeader, ) -> Result<(), ShadowServiceErrorV1> { - self.send(ShadowServiceMessageV1::LocalCarrier( - ShadowLocalCarrierV1::from_direct_header(header), - )) + self.local_application(header, None) + } + + /// Coalescing producer boundary for a standalone embedded application. + /// The optional payload is availability data only; its header remains the + /// sole identity and is commitment-checked inside the actor before use. + pub(crate) fn local_application( + &self, + header: &RbcCanonicalHeader, + application_payload: Option>, + ) -> Result<(), ShadowServiceErrorV1> { + if let Some(reason) = *self.invalidated_by_overload.lock() { + return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); + } + if let Some(payload) = &application_payload { + validate_application_payload_size(payload)?; + } + let local = + ShadowLocalCarrierV1::from_direct_header_with_payload(header, application_payload); + let mut desired = self.desired_local_applications.lock(); + if let Some(existing) = desired.get_mut(&local.round) { + if !existing.same_application(&local) { + return Err(ShadowServiceErrorV1::ConflictingLocalHeader(local.round)); + } + merge_application_payload( + &mut existing.application_payload, + local.application_payload, + existing.application_header.reference(), + )?; + existing.acknowledge_assignment |= local.acknowledge_assignment; + return Ok(()); + } + if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + return Err(ShadowServiceErrorV1::ApplicationStateCapacity { + capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, + }); + } + desired.insert(local.round, local); + drop(desired); + match self + .sender + .try_send(ShadowServiceMessageV1::LocalApplicationsChanged) + { + Ok(()) | Err(TrySendError::Full(_)) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), + } } + #[cfg(test)] pub(crate) fn carrier( &self, peer: AuthorityIndex, @@ -207,9 +466,35 @@ impl StarfishRbcDagShadowServiceHandleV1 { envelope.authentication_sidecar.len(), self.max_sidecar_size, )?; + if let Some(payload) = &envelope.application_payload { + validate_application_payload_size(payload)?; + } self.send(ShadowServiceMessageV1::Carrier { peer, envelope }) } + pub(crate) async fn carrier_reliably( + &self, + peer: AuthorityIndex, + envelope: RbcDagShadowCarrier, + ) -> Result<(), ShadowServiceErrorV1> { + validate_wire_size( + "carrier", + envelope.canonical_carrier.len(), + MAX_CARRIER_CONTENT_SIZE_V1, + )?; + validate_wire_size( + "authentication sidecar", + envelope.authentication_sidecar.len(), + self.max_sidecar_size, + )?; + if let Some(payload) = &envelope.application_payload { + validate_application_payload_size(payload)?; + } + self.send_reliably(ShadowServiceMessageV1::Carrier { peer, envelope }) + .await + } + + #[cfg(test)] pub(crate) fn carrier_request( &self, peer: AuthorityIndex, @@ -218,6 +503,16 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierRequest { peer, reference }) } + pub(crate) async fn carrier_request_reliably( + &self, + peer: AuthorityIndex, + reference: BlockReference, + ) -> Result<(), ShadowServiceErrorV1> { + self.send_reliably(ShadowServiceMessageV1::CarrierRequest { peer, reference }) + .await + } + + #[cfg(test)] pub(crate) fn carrier_response( &self, peer: AuthorityIndex, @@ -231,6 +526,21 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierResponse { peer, response }) } + pub(crate) async fn carrier_response_reliably( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierResponse, + ) -> Result<(), ShadowServiceErrorV1> { + validate_wire_size( + "carrier response", + response.canonical_carrier.len(), + MAX_CARRIER_CONTENT_SIZE_V1, + )?; + self.send_reliably(ShadowServiceMessageV1::CarrierResponse { peer, response }) + .await + } + + #[cfg(test)] pub(crate) fn carrier_sync_request( &self, peer: AuthorityIndex, @@ -242,14 +552,46 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierSyncRequest { peer, request }) } + pub(crate) async fn carrier_sync_request_reliably( + &self, + peer: AuthorityIndex, + request: RbcDagShadowCarrierSyncRequest, + ) -> Result<(), ShadowServiceErrorV1> { + if !self.mode.is_autonomous() { + return Ok(()); + } + self.send_reliably(ShadowServiceMessageV1::CarrierSyncRequest { peer, request }) + .await + } + + #[cfg(test)] pub(crate) fn carrier_sync_response( &self, peer: AuthorityIndex, response: RbcDagShadowCarrierSyncResponse, + ) -> Result<(), ShadowServiceErrorV1> { + self.enqueue_carrier_sync_response(peer, response) + } + + pub(crate) async fn carrier_sync_response_reliably( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierSyncResponse, + ) -> Result<(), ShadowServiceErrorV1> { + self.enqueue_carrier_sync_response(peer, response) + } + + fn enqueue_carrier_sync_response( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierSyncResponse, ) -> Result<(), ShadowServiceErrorV1> { if !self.mode.is_autonomous() { return Ok(()); } + if peer as usize >= self.committee_size { + return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); + } validate_wire_size( "carrier sync response", response.canonical_carrier.len(), @@ -260,7 +602,116 @@ impl StarfishRbcDagShadowServiceHandleV1 { response.authentication_sidecar.len(), self.max_sidecar_size, )?; - self.send(ShadowServiceMessageV1::CarrierSyncResponse { peer, response }) + if response.author != peer { + return Err(ShadowServiceErrorV1::UnexpectedSyncResponse { + author: response.author, + round: response.round, + }); + } + + let slot = (response.round, response.author); + let mut desired = self.desired_carrier_sync_responses.lock(); + match desired.get(&slot) { + Some((existing_peer, existing)) if *existing_peer == peer && *existing == response => { + return Ok(()); + } + Some(_) => { + return Err(ShadowServiceErrorV1::UnexpectedSyncResponse { + author: response.author, + round: response.round, + }); + } + None if desired.len() >= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 => { + return Err(ShadowServiceErrorV1::CarrierSyncResponseCapacity { + capacity: SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, + }); + } + None => {} + } + desired.insert(slot, (peer, response)); + drop(desired); + match self + .sender + .try_send(ShadowServiceMessageV1::CarrierSyncResponsesChanged) + { + Ok(()) | Err(TrySendError::Full(_)) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), + } + } + + #[cfg(test)] + pub(crate) fn application_payload_request( + &self, + peer: AuthorityIndex, + application: BlockReference, + ) -> Result<(), ShadowServiceErrorV1> { + self.send(ShadowServiceMessageV1::ApplicationPayloadRequest { peer, application }) + } + + pub(crate) async fn application_payload_request_reliably( + &self, + peer: AuthorityIndex, + application: BlockReference, + ) -> Result<(), ShadowServiceErrorV1> { + self.send_reliably(ShadowServiceMessageV1::ApplicationPayloadRequest { peer, application }) + .await + } + + #[cfg(test)] + pub(crate) fn application_payload_response( + &self, + peer: AuthorityIndex, + response: RbcDagApplicationPayloadResponse, + ) -> Result<(), ShadowServiceErrorV1> { + validate_application_payload_size(&response.transaction_data)?; + self.send(ShadowServiceMessageV1::ApplicationPayloadResponse { peer, response }) + } + + pub(crate) async fn application_payload_response_reliably( + &self, + peer: AuthorityIndex, + response: RbcDagApplicationPayloadResponse, + ) -> Result<(), ShadowServiceErrorV1> { + validate_application_payload_size(&response.transaction_data)?; + self.send_reliably(ShadowServiceMessageV1::ApplicationPayloadResponse { peer, response }) + .await + } + + /// Trusted callback from the sole transaction-commitment verifier. The + /// actor accepts it only for an already-authorized application reference; + /// network payload bytes can never call this path directly. + pub(crate) fn verified_application_payload( + &self, + application: BlockReference, + payload: Arc, + ) -> Result<(), ShadowServiceErrorV1> { + if let Some(reason) = *self.invalidated_by_overload.lock() { + return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); + } + validate_application_payload_size(&payload)?; + let mut desired = self.desired_verified_application_payloads.lock(); + if let Some(existing) = desired.get(&application) { + if !application_payloads_equal(Some(existing.as_ref()), Some(payload.as_ref())) { + return Err(ShadowServiceErrorV1::ConflictingApplicationPayload( + application, + )); + } + return Ok(()); + } + if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + return Err(ShadowServiceErrorV1::ApplicationStateCapacity { + capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, + }); + } + desired.insert(application, payload); + drop(desired); + match self + .sender + .try_send(ShadowServiceMessageV1::VerifiedApplicationPayloadsChanged) + { + Ok(()) | Err(TrySendError::Full(_)) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), + } } pub(crate) fn direct_delivered( @@ -333,6 +784,39 @@ impl StarfishRbcDagShadowServiceHandleV1 { receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped)? } + /// Release a coordinated autonomous-clock start barrier. This operation + /// is idempotent: after the first successful activation, later callers + /// receive an acknowledgment without resetting the heartbeat epoch. + pub(crate) async fn activate_clock(&self) -> Result<(), ShadowServiceErrorV1> { + if !self.mode.is_autonomous() { + return Ok(()); + } + let (reply, receiver) = oneshot::channel(); + self.send_reliably(ShadowServiceMessageV1::ActivateClock(reply)) + .await?; + receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped) + } + + #[cfg(test)] + async fn inspect_rbc_progress(&self) -> Result<(usize, usize), ShadowServiceErrorV1> { + let (reply, receiver) = oneshot::channel(); + self.sender + .send(ShadowServiceMessageV1::InspectRbcProgress(reply)) + .await + .map_err(|_| ShadowServiceErrorV1::Stopped)?; + receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped) + } + + #[cfg(test)] + async fn inspect_carrier_sync(&self) -> Result { + let (reply, receiver) = oneshot::channel(); + self.sender + .send(ShadowServiceMessageV1::InspectCarrierSync(reply)) + .await + .map_err(|_| ShadowServiceErrorV1::Stopped)?; + receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped) + } + fn update_peer( &self, peer: AuthorityIndex, @@ -362,29 +846,52 @@ impl StarfishRbcDagShadowServiceHandleV1 { } impl ShadowServiceMessageV1 { + #[cfg(test)] fn kind(&self) -> &'static str { match self { - Self::LocalCarrier(_) => "local", + Self::ActivateClock(_) => "activate_clock", + Self::LocalApplicationsChanged => "local_applications_changed", Self::Carrier { .. } => "carrier", Self::CarrierRequest { .. } => "carrier_request", Self::CarrierResponse { .. } => "carrier_response", Self::CarrierSyncRequest { .. } => "carrier_sync_request", - Self::CarrierSyncResponse { .. } => "carrier_sync_response", + Self::CarrierSyncResponsesChanged => "carrier_sync_responses_changed", + Self::ApplicationPayloadRequest { .. } => "application_payload_request", + Self::ApplicationPayloadResponse { .. } => "application_payload_response", + Self::VerifiedApplicationPayloadsChanged => "verified_application_payloads_changed", Self::DirectDeliveriesChanged => "direct_deliveries_changed", Self::TopologyChanged => "topology_changed", Self::RetryRecovery => "recovery_retry", Self::HeartbeatTick => "heartbeat_tick", + Self::NormalCarrierDeadline { .. } => "normal_carrier_deadline", + Self::ConsensusTimeoutDeadline { .. } => "consensus_timeout_deadline", Self::DataAvailabilityChanged => "data_availability_changed", + Self::InspectRbcProgress(_) => "inspect_rbc_progress", + Self::InspectCarrierSync(_) => "inspect_carrier_sync", Self::Shutdown(_) => "shutdown", } } } +/// Exact protocol fact permitting an embedded application header to leave the +/// shadow actor. Payload availability is deliberately absent from this enum: +/// bytes can accompany authority, but can never create it. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ShadowApplicationAuthorizationBasisV1 { + LocallyFixed, + ReceiverAuthenticated, + Delivered, +} + #[derive(Debug)] pub(crate) enum ShadowServiceEventV1 { Ready { autonomous_clock: bool, }, + /// The autonomous actor has crossed its one-way coordinated-start + /// barrier. Local carrier creation and exact repair are enabled, and the + /// first physical heartbeat is one full interval after this event. + ClockActivated, ClockState { open_round: RoundNumber, phase_backlog: usize, @@ -406,6 +913,21 @@ pub(crate) enum ShadowServiceEventV1 { carrier: BlockReference, header: RbcCanonicalHeader, }, + /// An application header whose enclosing carrier has an exact local, + /// receiver-authenticated, or delivered authorization basis. A later + /// repeat may enrich an earlier header-only event with its verified + /// transaction payload. + AuthorizedApplicationObserved { + carrier: BlockReference, + header: RbcCanonicalHeader, + payload: Option>, + authorization_basis: ShadowApplicationAuthorizationBasisV1, + }, + /// The exact direct application header has been durably assigned to one + /// local carrier. In embedded-authority mode this is the flow-control + /// acknowledgment that permits the core to produce the next application + /// header; it is not a delivery or commit certificate. + ApplicationAssigned(BlockReference), VertexProjected(ConsensusVertexReference), LeaderDecided(ProjectionDecisionV1), FrontierCommitted(CommittedFrontierDeltaV1), @@ -435,6 +957,7 @@ pub(crate) enum ShadowServiceErrorV1 { Shadow(ShadowErrorV1), StartTask(tokio::task::JoinError), Stopped, + #[cfg(test)] Overloaded { kind: &'static str, capacity: usize, @@ -455,6 +978,18 @@ pub(crate) enum ShadowServiceErrorV1 { UnknownAuthority(AuthorityIndex), Loopback(AuthorityIndex), ConflictingLocalHeader(RoundNumber), + ConflictingApplicationPayload(BlockReference), + ApplicationStateCapacity { + capacity: usize, + }, + CarrierSyncResponseCapacity { + capacity: usize, + }, + ApplicationPayloadSerialization(String), + ApplicationPayloadResponseFromUnexpectedPeer { + peer: AuthorityIndex, + application: BlockReference, + }, MissingRecoveredLocalHeader(RoundNumber), RecoveredLocalHeaderMismatch(RoundNumber), AutonomousWalContainsInvalidCarrier(RoundNumber), @@ -494,6 +1029,7 @@ impl fmt::Display for ShadowServiceErrorV1 { "Starfish-RBC-DAG shadow startup task failed: {error}" ), Self::Stopped => formatter.write_str("Starfish-RBC-DAG shadow service stopped"), + #[cfg(test)] Self::Overloaded { kind, capacity } => write!( formatter, "Starfish-RBC-DAG shadow {kind} input was dropped because the queue is full \ @@ -530,6 +1066,26 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "conflicting direct headers supplied for queued shadow round {round}" ), + Self::ConflictingApplicationPayload(application) => write!( + formatter, + "conflicting transaction payloads supplied for embedded application {application}" + ), + Self::ApplicationStateCapacity { capacity } => write!( + formatter, + "embedded application state reached its bounded capacity of {capacity} entries" + ), + Self::CarrierSyncResponseCapacity { capacity } => write!( + formatter, + "exact carrier-sync response state reached its bounded capacity of {capacity} slots" + ), + Self::ApplicationPayloadSerialization(error) => write!( + formatter, + "embedded application payload could not be size-checked: {error}" + ), + Self::ApplicationPayloadResponseFromUnexpectedPeer { peer, application } => write!( + formatter, + "embedded application payload response for {application} came from unrequested authority {peer}" + ), Self::MissingRecoveredLocalHeader(round) => write!( formatter, "persisted shadow carrier at round {round} has no matching recovered direct header" @@ -596,6 +1152,7 @@ impl From for ShadowServiceErrorV1 { } } +#[cfg(test)] pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, @@ -621,18 +1178,22 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( recovered_local_headers, ShadowServiceModeV1::DirectMirror, wal_sync_policy, + None, + None, + true, ) } -pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, own_authority: AuthorityIndex, context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, - heartbeat_interval: Duration, wal_sync_policy: ShadowWalSyncPolicyV1, + metrics: Arc, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -641,9 +1202,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( ), ShadowServiceErrorV1, > { - if heartbeat_interval.is_zero() { - return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); - } start_starfish_rbc_dag_shadow_service_with_mode_v1( path, committee, @@ -651,12 +1209,238 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( context, authorizer, recovered_local_headers, - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + ShadowServiceModeV1::DirectMirror, wal_sync_policy, + Some(metrics), + None, + true, ) } -fn start_starfish_rbc_dag_shadow_service_with_mode_v1( +#[cfg(test)] +pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, + heartbeat_interval: Duration, + wal_sync_policy: ShadowWalSyncPolicyV1, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, +> { + if heartbeat_interval.is_zero() { + return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); + } + start_starfish_rbc_dag_shadow_service_with_mode_v1( + path, + committee, + own_authority, + context, + authorizer, + recovered_local_headers, + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + wal_sync_policy, + None, + None, + true, + ) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, + heartbeat_interval: Duration, + wal_sync_policy: ShadowWalSyncPolicyV1, + metrics: Arc, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, +> { + if heartbeat_interval.is_zero() { + return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); + } + start_starfish_rbc_dag_shadow_service_with_mode_v1( + path, + committee, + own_authority, + context, + authorizer, + recovered_local_headers, + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + wal_sync_policy, + Some(metrics), + None, + true, + ) +} + +/// Start an autonomous runtime whose protocol clock remains paused after WAL +/// open/replay and `Ready`. The caller must invoke +/// [`StarfishRbcDagShadowServiceHandleV1::activate_clock`] after its external +/// startup barrier is satisfied. Ordinary start APIs remain active-by-default. +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_paused_with_metrics_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, + heartbeat_interval: Duration, + wal_sync_policy: ShadowWalSyncPolicyV1, + metrics: Arc, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, +> { + if heartbeat_interval.is_zero() { + return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); + } + start_starfish_rbc_dag_shadow_service_with_mode_v1( + path, + committee, + own_authority, + context, + authorizer, + recovered_local_headers, + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + wal_sync_policy, + Some(metrics), + None, + false, + ) +} + +/// Start the embedded-authority runtime with Core's exact durable recovery +/// cursor. Unlike the compatibility autonomous wrappers, this is the +/// production restart API: it fails closed unless the actor WAL reconciles +/// the cursor and its unapplied suffix is bounded. +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1( + path: impl AsRef, + committee: RbcDagCommitteeContextV1, + own_authority: AuthorityIndex, + context: RbcDagContextV1, + authorizer: ShadowAuthorizerV1, + recovered_local_headers: Vec, + heartbeat_interval: Duration, + wal_sync_policy: ShadowWalSyncPolicyV1, + metrics: Arc, + recovery_cursor: Option, + clock_starts_active: bool, +) -> Result< + ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ), + ShadowServiceErrorV1, +> { + if heartbeat_interval.is_zero() { + return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); + } + start_starfish_rbc_dag_shadow_service_with_mode_v1( + path, + committee, + own_authority, + context, + authorizer, + recovered_local_headers, + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + wal_sync_policy, + Some(metrics), + recovery_cursor, + clock_starts_active, + ) +} + +fn spawn_consensus_timeout_deadline_task( + mut deadline_rx: watch::Receiver>, + timeout_tx: mpsc::WeakSender, +) { + tokio::spawn(async move { + loop { + if deadline_rx.changed().await.is_err() { + return; + } + let Some(mut target) = *deadline_rx.borrow_and_update() else { + continue; + }; + 'scheduled: loop { + tokio::select! { + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(target.deadline)) => { + if *deadline_rx.borrow() != Some(target) { + break 'scheduled; + } + let Some(sender) = timeout_tx.upgrade() else { + return; + }; + let message = ShadowServiceMessageV1::ConsensusTimeoutDeadline { + generation: target.generation, + slot: target.slot, + }; + // A full actor FIFO must not pin a strong sender after + // the actor cancels/replaces this deadline or exits. + // The watch branch aborts the pending send; any wake + // already queued is rejected by its generation. + tokio::select! { + biased; + changed = deadline_rx.changed() => { + if changed.is_err() { + return; + } + match *deadline_rx.borrow_and_update() { + Some(replacement) => { + target = replacement; + continue 'scheduled; + } + None => break 'scheduled, + } + } + sent = sender.send(message) => { + if sent.is_err() { + return; + } + break 'scheduled; + } + } + } + changed = deadline_rx.changed() => { + if changed.is_err() { + return; + } + match *deadline_rx.borrow_and_update() { + Some(replacement) => target = replacement, + None => break 'scheduled, + } + } + } + } + } + }); +} + +fn start_starfish_rbc_dag_shadow_service_with_mode_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, own_authority: AuthorityIndex, @@ -665,6 +1449,9 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( recovered_local_headers: Vec, mode: ShadowServiceModeV1, wal_sync_policy: ShadowWalSyncPolicyV1, + metrics: Option>, + recovery_cursor: Option, + clock_starts_active: bool, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -680,7 +1467,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let path = path.as_ref().to_path_buf(); let mut pending_local = BTreeMap::new(); for header in recovered_local_headers { - let local = ShadowLocalCarrierV1::from_direct_header(&header); + let local = ShadowLocalCarrierV1::from_recovered_direct_header(&header); if local.author != own_authority { return Err(ShadowServiceErrorV1::LocalHeaderAuthority { expected: own_authority, @@ -697,11 +1484,24 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let (message_tx, message_rx) = mpsc::channel(input_capacity); let (event_tx, event_rx) = mpsc::channel(SHADOW_SERVICE_EVENT_CAPACITY_V1); let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); + let desired_local_applications = Arc::new(Mutex::new(BTreeMap::new())); + let desired_carrier_sync_responses = Arc::new(Mutex::new(BTreeMap::new())); + let desired_verified_application_payloads = Arc::new(Mutex::new(BTreeMap::new())); let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); let desired_available_applications = Arc::new(Mutex::new(BTreeSet::new())); let invalidated_by_overload = Arc::new(Mutex::new(None)); let retry_notification_pending = Arc::new(AtomicBool::new(false)); let heartbeat_notification_pending = Arc::new(AtomicBool::new(false)); + // The physical heartbeat is deliberately created only after the actor + // crosses its one-way activation barrier. In paused benchmark startup, + // validators can therefore spend arbitrarily different amounts of time + // opening WALs and establishing topology without inheriting staggered + // timer phases or accumulating missed ticks. + let (clock_activation_tx, mut clock_activation_rx) = watch::channel(false); + let (normal_carrier_deadline_tx, mut normal_carrier_deadline_rx) = + watch::channel(None::); + let (consensus_timeout_deadline_tx, consensus_timeout_deadline_rx) = + watch::channel(None::); let retry_tx = message_tx.downgrade(); let retry_pending = Arc::clone(&retry_notification_pending); tokio::spawn(async move { @@ -730,12 +1530,16 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let heartbeat_tx = message_tx.downgrade(); let heartbeat_pending = Arc::clone(&heartbeat_notification_pending); tokio::spawn(async move { - let mut interval = tokio::time::interval(heartbeat_interval); + if clock_activation_rx + .wait_for(|active| *active) + .await + .is_err() + { + return; + } + let first_tick = tokio::time::Instant::now() + heartbeat_interval; + let mut interval = tokio::time::interval_at(first_tick, heartbeat_interval); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // Give startup/WAL replay one full interval before the first - // carrier. A missed/full notification is harmless: a later tick - // retries the still-open local slot. - interval.tick().await; loop { interval.tick().await; let Some(heartbeat_tx) = heartbeat_tx.upgrade() else { @@ -757,23 +1561,86 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }); } + // One persistent, generation-tagged deadline task replaces per-attempt + // sleeps. Repeated normal creation requests only replace the desired + // deadline in the watch channel; a stale queued wake is harmless. + let normal_carrier_tx = message_tx.downgrade(); + tokio::spawn(async move { + loop { + if normal_carrier_deadline_rx.changed().await.is_err() { + return; + } + let Some(mut target) = *normal_carrier_deadline_rx.borrow_and_update() else { + continue; + }; + loop { + tokio::select! { + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(target.deadline)) => { + if *normal_carrier_deadline_rx.borrow() != Some(target) { + break; + } + let Some(sender) = normal_carrier_tx.upgrade() else { + return; + }; + if sender + .send(ShadowServiceMessageV1::NormalCarrierDeadline { + generation: target.generation, + }) + .await + .is_err() + { + return; + } + break; + } + changed = normal_carrier_deadline_rx.changed() => { + if changed.is_err() { + return; + } + match *normal_carrier_deadline_rx.borrow_and_update() { + Some(replacement) => target = replacement, + None => break, + } + } + } + } + } + }); + spawn_consensus_timeout_deadline_task(consensus_timeout_deadline_rx, message_tx.downgrade()); let startup_events = event_tx.clone(); let actor_desired_topology = Arc::clone(&desired_topology); + let actor_desired_local_applications = Arc::clone(&desired_local_applications); + let actor_desired_carrier_sync_responses = Arc::clone(&desired_carrier_sync_responses); + let actor_desired_verified_application_payloads = + Arc::clone(&desired_verified_application_payloads); let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); let actor_desired_available_applications = Arc::clone(&desired_available_applications); let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); let actor_retry_notification_pending = Arc::clone(&retry_notification_pending); let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); + let actor_committee = committee.clone(); let task = tokio::spawn(async move { let opened = tokio::task::spawn_blocking(move || { - StarfishRbcDagShadowV1::open_with_wal_sync_policy( - path, - committee, - own_authority, - context, - authorizer, - wal_sync_policy, - ) + if mode.is_autonomous() { + StarfishRbcDagShadowV1::open_authoritative_with_wal_sync_policy( + path, + committee, + own_authority, + context, + authorizer, + wal_sync_policy, + recovery_cursor, + ) + } else { + StarfishRbcDagShadowV1::open_with_wal_sync_policy( + path, + committee, + own_authority, + context, + authorizer, + wal_sync_policy, + ) + } }) .await; let (core, open_report) = match opened { @@ -948,11 +1815,8 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }; let recovered_shadow_deliveries = reported_shadow_deliveries.clone(); - let reported_application_deliveries = match core.delivered_application_headers() { - Ok(headers) => headers - .into_iter() - .map(|(_, header)| header.reference()) - .collect(), + let recovered_application_headers = match core.delivered_application_headers() { + Ok(headers) => headers, Err(error) => { let _ = startup_events .send(ShadowServiceEventV1::Rejected { @@ -963,21 +1827,57 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( return; } }; + let reported_application_deliveries = recovered_application_headers + .iter() + .map(|(_, header)| header.reference()) + .collect(); + let mut authorized_applications = BTreeMap::new(); + for (carrier, header) in recovered_application_headers + .into_iter() + .rev() + .take(SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1) + { + authorized_applications.insert( + header.reference(), + AuthorizedApplicationStateV1 { + carrier, + header, + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + verified_payload: None, + observed_payload: None, + observed_payload_at: None, + holders: BTreeSet::new(), + request_last_attempt: BTreeMap::new(), + emitted_header: false, + emitted_observed_payload: false, + }, + ); + } let reported_shadow_delivery_slots = reported_shadow_deliveries .iter() .map(delivery_slot) .collect(); let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); let sync_round = core.local_carrier_round(); + let consensus_pacemaker = ConsensusPacemakerV1::new(core.next_local_consensus_round()); let state = ShadowServiceStateV1 { core, + committee: actor_committee, mode, + clock_active: false, + clock_activation_tx, wal_sync_policy, + metrics, own_authority, committee_size, events: event_tx, connected: BTreeSet::new(), + catch_up_hint_high_water: BTreeMap::new(), + far_future_hint_high_water: BTreeMap::new(), desired_topology: actor_desired_topology, + desired_local_applications: actor_desired_local_applications, + desired_carrier_sync_responses: actor_desired_carrier_sync_responses, + desired_verified_application_payloads: actor_desired_verified_application_payloads, desired_direct_deliveries: actor_desired_direct_deliveries, desired_available_applications: actor_desired_available_applications, observed_topology: BTreeMap::new(), @@ -989,10 +1889,33 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), sync_last_served: BTreeMap::new(), + authorized_applications, + quarantined_application_payloads: BTreeMap::new(), + payload_last_served: BTreeMap::new(), sync_round, sync_round_opened_at: Instant::now(), sync_catch_up: false, - sync_used_in_open_round: false, + sync_catch_up_limit_future: false, + sync_catch_up_target: None, + #[cfg(test)] + sync_max_outstanding: 0, + #[cfg(test)] + sync_max_desired_responses: 0, + awaiting_application_submission: false, + consensus_pacemaker, + normal_carrier_min_spacing: mode + .heartbeat_interval() + .and_then(|interval| interval.checked_div(SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1)) + .unwrap_or_default() + .max(SHADOW_NORMAL_CARRIER_MIN_SPACING_V1), + normal_carrier_next_allowed_at: None, + normal_carrier_requested: false, + normal_carrier_generation: 0, + normal_carrier_deadline: None, + normal_carrier_deadline_tx, + consensus_timeout_generation: 0, + consensus_timeout_deadline: None, + consensus_timeout_deadline_tx, retry_notification_pending: actor_retry_notification_pending, heartbeat_notification_pending: actor_heartbeat_notification_pending, direct_deliveries: BTreeSet::new(), @@ -1006,7 +1929,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( fatal: false, }; if let Err(error) = tokio::task::spawn_blocking(move || { - run_shadow_service(state, message_rx, open_report); + run_shadow_service(state, message_rx, open_report, clock_starts_active); }) .await { @@ -1024,9 +1947,13 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( max_sidecar_size, own_authority, committee_size, + #[cfg(test)] input_capacity, mode, desired_topology, + desired_local_applications, + desired_carrier_sync_responses, + desired_verified_application_payloads, desired_direct_deliveries, desired_available_applications, invalidated_by_overload, @@ -1100,15 +2027,126 @@ impl ShadowComparisonBacklogV1 { } } +#[derive(Clone, Debug)] +struct AuthorizedApplicationStateV1 { + carrier: BlockReference, + header: RbcCanonicalHeader, + authorization_basis: ShadowApplicationAuthorizationBasisV1, + verified_payload: Option>, + observed_payload: Option>, + observed_payload_at: Option, + holders: BTreeSet, + request_last_attempt: BTreeMap, + emitted_header: bool, + emitted_observed_payload: bool, +} + +#[derive(Clone, Debug)] +struct QuarantinedApplicationPayloadV1 { + application: BlockReference, + holder: AuthorityIndex, + payload: Option>, +} + +/// Per-requester exact-slot replay limiter. A lagging honest peer may consume +/// a bounded set of distinct rounds faster than the healthy carrier clock, +/// independent of response/request ordering. Replays of a round already +/// served in the current interval and a seventeenth distinct round are +/// throttled. +#[derive(Clone, Debug)] +struct CarrierSyncServeWindowV1 { + started_at: Instant, + served_rounds: BTreeSet, +} + +impl CarrierSyncServeWindowV1 { + fn first(round: RoundNumber, now: Instant) -> Self { + let mut served_rounds = BTreeSet::new(); + served_rounds.insert(round); + Self { + started_at: now, + served_rounds, + } + } + + fn permits(&mut self, round: RoundNumber, now: Instant) -> bool { + if now.saturating_duration_since(self.started_at) >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 { + self.started_at = now; + self.served_rounds.clear(); + } + if self.served_rounds.contains(&round) + || self.served_rounds.len() >= SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1 + { + return false; + } + self.served_rounds.insert(round) + } +} + +fn carrier_sync_pipeline_depth(committee_size: usize) -> usize { + let remote_authors = committee_size.saturating_sub(1); + if remote_authors == 0 { + return 0; + } + SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1 + .min(SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 / remote_authors) +} + +/// Produce the bounded round-major repair window. `target == None` is the +/// healthy/moderate single-round path; a catch-up target opens only enough +/// consecutive rounds to keep all `n - 1` authors inside the global credit. +fn carrier_sync_pipeline_slots( + current_round: RoundNumber, + target: Option, + committee_size: usize, + connected: &BTreeSet, +) -> Vec { + let depth = carrier_sync_pipeline_depth(committee_size); + if depth == 0 { + return Vec::new(); + } + let depth_delta = RoundNumber::try_from(depth.saturating_sub(1)).unwrap_or(RoundNumber::MAX); + let last_round = current_round + .saturating_add(depth_delta) + .min(target.unwrap_or(current_round)); + if last_round < current_round { + return Vec::new(); + } + (current_round..=last_round) + .flat_map(|round| connected.iter().copied().map(move |author| (round, author))) + .take(SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1) + .collect() +} + struct ShadowServiceStateV1 { core: StarfishRbcDagShadowV1, + committee: RbcDagCommitteeContextV1, mode: ShadowServiceModeV1, + /// One-way coordinated-start latch for the autonomous protocol clock. + /// Mirror mode never consults this flag. + clock_active: bool, + clock_activation_tx: watch::Sender, wal_sync_policy: ShadowWalSyncPolicyV1, + metrics: Option>, own_authority: AuthorityIndex, committee_size: usize, events: mpsc::Sender, connected: BTreeSet, + /// Distinct transport-authenticated peers that exposed a carrier beyond + /// the local admission/retention horizon. Validity stake is required + /// before this performance-only hint can enable aggressive catch-up, so a + /// Byzantine minority cannot force the node into repair mode. + catch_up_hint_high_water: BTreeMap, + /// Subset of catch-up hints that were outside the authenticated 64-round + /// retention horizon. Only validity stake in this set disables proactive + /// future buffering; ordinary small skew keeps the useful retained tail. + far_future_hint_high_water: BTreeMap, desired_topology: Arc>>, + desired_local_applications: Arc>>, + desired_carrier_sync_responses: + Arc>>, + desired_verified_application_payloads: + Arc>>>, desired_direct_deliveries: Arc>>, desired_available_applications: Arc>>, observed_topology: BTreeMap, @@ -1118,12 +2156,35 @@ struct ShadowServiceStateV1 { pending_data_availability: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, - sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, - sync_last_served: BTreeMap, + sync_last_attempt: BTreeMap, + sync_last_served: BTreeMap, + authorized_applications: BTreeMap, + quarantined_application_payloads: BTreeMap, + payload_last_served: BTreeMap, sync_round: RoundNumber, sync_round_opened_at: Instant, sync_catch_up: bool, - sync_used_in_open_round: bool, + sync_catch_up_limit_future: bool, + sync_catch_up_target: Option, + #[cfg(test)] + sync_max_outstanding: usize, + #[cfg(test)] + sync_max_desired_responses: usize, + /// A successful application-carrier assignment just released the core's + /// one-outstanding producer gate. Give that exact producer one actor turn + /// to submit its successor before C3 spends the next physical slot on a + /// control heartbeat. Maintenance/heartbeat messages bound the wait. + awaiting_application_submission: bool, + consensus_pacemaker: ConsensusPacemakerV1, + normal_carrier_min_spacing: Duration, + normal_carrier_next_allowed_at: Option, + normal_carrier_requested: bool, + normal_carrier_generation: u64, + normal_carrier_deadline: Option, + normal_carrier_deadline_tx: watch::Sender>, + consensus_timeout_generation: u64, + consensus_timeout_deadline: Option, + consensus_timeout_deadline_tx: watch::Sender>, retry_notification_pending: Arc, heartbeat_notification_pending: Arc, direct_deliveries: BTreeSet, @@ -1142,6 +2203,51 @@ impl ShadowServiceStateV1 { let _ = self.events.blocking_send(event); } + fn activate_clock(&mut self) { + if !self.mode.is_autonomous() || self.clock_active { + return; + } + self.clock_active = true; + let now = Instant::now(); + self.sync_round = self.core.local_carrier_round(); + self.sync_round_opened_at = now; + self.cancel_consensus_timeout_deadline(); + self.consensus_pacemaker = + ConsensusPacemakerV1::new(self.core.next_local_consensus_round()); + self.awaiting_application_submission = false; + self.heartbeat_notification_pending + .store(false, Ordering::Release); + + // Queue the ordered activation observation before releasing the timer + // task. Even if the event channel is temporarily full, no heartbeat + // can be generated ahead of `ClockActivated`. + self.emit(ShadowServiceEventV1::ClockActivated); + self.clock_activation_tx.send_replace(true); + self.emit_clock_state(); + } + + fn emit_recovered_authorized_applications(&mut self) { + let applications = self + .authorized_applications + .keys() + .copied() + .collect::>(); + for application in applications { + self.emit_authorized_application_if_new(application); + } + } + + fn emit_autonomous_recovery_and_ready(&mut self, open_report: &ShadowOpenReportV1) { + self.emit_recovered_authorized_applications(); + for delta in open_report.recovered_committed_frontiers() { + self.emit(ShadowServiceEventV1::FrontierCommitted(delta.clone())); + } + self.process_effects(open_report.recovery_effects().to_vec()); + self.emit(ShadowServiceEventV1::Ready { + autonomous_clock: true, + }); + } + fn reject(&self, peer: Option, error: impl fmt::Display) { self.emit(ShadowServiceEventV1::Rejected { peer, @@ -1165,6 +2271,7 @@ impl ShadowServiceStateV1 { if !self.mode.is_autonomous() { return; } + self.record_pipeline_state(); self.emit(ShadowServiceEventV1::ClockState { open_round: self.core.local_carrier_round(), phase_backlog: self.core.pending_phase_backlog_len(), @@ -1174,6 +2281,22 @@ impl ShadowServiceStateV1 { }); } + fn record_pipeline_state(&self) { + let Some(metrics) = &self.metrics else { + return; + }; + let projection = self.core.projection_runtime_snapshot(); + metrics.set_starfish_rbc_dag_pipeline_state( + self.pending_local.len(), + projection.pending_candidates, + projection.highest_projected_round, + projection.next_undecided_round, + projection.next_undecided_projected_stake, + projection.last_committed_round, + projection.hol_reason.metric_label(), + ); + } + fn validate_peer(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { if peer as usize >= self.committee_size { return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); @@ -1184,24 +2307,536 @@ impl ShadowServiceStateV1 { Ok(()) } - fn reconcile_topology(&mut self) { - let desired = self.desired_topology.lock().clone(); - let mut newly_connected = Vec::new(); - for (peer, state) in &desired { - if self.observed_topology.get(peer) == Some(state) { - continue; - } - self.recovery_last_attempt - .retain(|(_, holder), _| holder != peer); - self.sync_last_attempt - .retain(|(author, _), _| author != peer); - self.sync_last_served - .retain(|requester, _| requester != peer); - if state.0 { - self.connected.insert(*peer); - newly_connected.push(*peer); - } else { + fn decode_application_carrier( + &self, + canonical_carrier: &[u8], + ) -> Result, ShadowErrorV1> { + let candidate = CandidateCarrierV1::decode_wire_with_committee( + canonical_carrier, + &self.committee, + None, + ) + .map_err(ShadowErrorV1::Carrier)?; + Ok(candidate + .header() + .application_header() + .cloned() + .map(|header| (candidate.reference(), header))) + } + + fn make_application_state_room(&mut self, application: BlockReference) { + if self.authorized_applications.contains_key(&application) + || self.authorized_applications.len() < SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + { + return; + } + if let Some((evicted, _)) = self.authorized_applications.pop_first() { + self.quarantined_application_payloads + .retain(|_, retained| retained.application != evicted); + } + } + + fn emit_authorized_application_if_new(&mut self, application: BlockReference) { + let event = self + .authorized_applications + .get_mut(&application) + .and_then(|state| { + let should_emit = !state.emitted_header + || (state.observed_payload.is_some() && !state.emitted_observed_payload); + should_emit.then(|| { + state.emitted_header = true; + state.emitted_observed_payload |= state.observed_payload.is_some(); + ShadowServiceEventV1::AuthorizedApplicationObserved { + carrier: state.carrier, + header: state.header.clone(), + payload: state + .observed_payload + .clone() + .or_else(|| state.verified_payload.clone()), + authorization_basis: state.authorization_basis, + } + }) + }); + if let Some(event) = event { + self.emit(event); + } + } + + fn authorize_application( + &mut self, + carrier: BlockReference, + header: RbcCanonicalHeader, + payload: Option>, + payload_is_verified: bool, + holder: Option, + authorization_basis: ShadowApplicationAuthorizationBasisV1, + ) -> Result<(), ShadowServiceErrorV1> { + let application = header.reference(); + self.make_application_state_room(application); + let state = self + .authorized_applications + .entry(application) + .or_insert_with(|| AuthorizedApplicationStateV1 { + carrier, + header: header.clone(), + authorization_basis, + verified_payload: None, + observed_payload: None, + observed_payload_at: None, + holders: BTreeSet::new(), + request_last_attempt: BTreeMap::new(), + emitted_header: false, + emitted_observed_payload: false, + }); + if state.header != header { + return Err(ShadowServiceErrorV1::ConflictingApplicationPayload( + application, + )); + } + if let Some(holder) = holder { + state.holders.insert(holder); + } + if payload_is_verified { + merge_application_payload(&mut state.verified_payload, payload, application)?; + } else { + let observed = payload.is_some(); + merge_application_payload(&mut state.observed_payload, payload, application)?; + if observed { + state.observed_payload_at = Some(Instant::now()); + } + } + self.emit_authorized_application_if_new(application); + self.flush_application_payload_requests(); + Ok(()) + } + + fn quarantine_application( + &mut self, + carrier: BlockReference, + application: BlockReference, + holder: AuthorityIndex, + payload: Option>, + ) -> Result<(), ShadowServiceErrorV1> { + if !self.quarantined_application_payloads.contains_key(&carrier) + && self.quarantined_application_payloads.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + { + self.quarantined_application_payloads.pop_first(); + } + let retained = self + .quarantined_application_payloads + .entry(carrier) + .or_insert(QuarantinedApplicationPayloadV1 { + application, + holder, + payload: None, + }); + if retained.application != application { + return Err(ShadowServiceErrorV1::ConflictingApplicationPayload( + application, + )); + } + merge_application_payload(&mut retained.payload, payload, application) + } + + fn observe_carrier_application( + &mut self, + peer: AuthorityIndex, + canonical_carrier: &[u8], + payload: Option>, + disposition: ShadowIngressDispositionV1, + ) -> Result<(), ShadowServiceErrorV1> { + if disposition == ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer { + return Ok(()); + } + let Some((carrier, header)) = self.decode_application_carrier(canonical_carrier)? else { + return Ok(()); + }; + let application = header.reference(); + let (observed_payload, payload_error) = match payload { + Some(payload) => match validate_application_payload_size(&payload) { + Ok(()) => (Some(payload), None), + Err(error) => (None, Some(error)), + }, + None => (None, None), + }; + match disposition { + ShadowIngressDispositionV1::Authenticated => self.authorize_application( + carrier, + header, + observed_payload, + false, + Some(peer), + ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, + )?, + ShadowIngressDispositionV1::CandidateRetained => { + self.quarantine_application(carrier, application, peer, observed_payload)? + } + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { + if self + .authorized_applications + .get(&application) + .is_some_and(|state| state.carrier == carrier) + { + self.authorize_application( + carrier, + header, + observed_payload, + false, + Some(peer), + ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, + )?; + } else if self.quarantined_application_payloads.contains_key(&carrier) { + self.quarantine_application(carrier, application, peer, observed_payload)?; + } + } + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => {} + } + if let Some(error) = payload_error { + return Err(error); + } + Ok(()) + } + + fn authorize_delivered_application( + &mut self, + carrier: BlockReference, + header: RbcCanonicalHeader, + ) -> Result<(), ShadowServiceErrorV1> { + let retained = self.quarantined_application_payloads.remove(&carrier); + let holder = retained.as_ref().map(|retained| retained.holder); + let payload = retained.and_then(|retained| retained.payload); + self.authorize_application( + carrier, + header, + payload, + false, + holder, + ShadowApplicationAuthorizationBasisV1::Delivered, + ) + } + + fn flush_application_payload_requests(&mut self) { + let now = Instant::now(); + let connected = &self.connected; + let mut requests = Vec::new(); + for (application, state) in &self.authorized_applications { + // The default commitment is the canonical proof of an empty + // application payload. DagState marks such a header available as + // soon as it is materialized, so there are no payload bytes to + // recover and no peer can produce a meaningful response. + if state.header.transactions_commitment() == TransactionsCommitment::default() + || state.verified_payload.is_some() + || state.observed_payload_at.is_some_and(|observed| { + now.saturating_duration_since(observed) + < SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 + }) + || state.request_last_attempt.values().any(|last| { + now.saturating_duration_since(*last) + < SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 + }) + { + continue; + } + let peer = state + .holders + .iter() + .copied() + .filter(|holder| connected.contains(holder)) + .chain(connected.iter().copied()) + .find(|holder| { + state.request_last_attempt.get(holder).is_none_or(|last| { + now.saturating_duration_since(*last) + >= SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 + }) + }); + if let Some(peer) = peer { + requests.push((*application, peer)); + } + } + for (application, peer) in requests { + let Some(state) = self.authorized_applications.get_mut(&application) else { + continue; + }; + state.observed_payload = None; + state.observed_payload_at = None; + state.emitted_observed_payload = false; + state.request_last_attempt.clear(); + state.request_last_attempt.insert(peer, now); + self.emit(ShadowServiceEventV1::Network { + recipient: peer, + message: NetworkMessage::RbcDagApplicationPayloadRequest(application), + }); + } + } + + fn handle_application_payload_request( + &mut self, + peer: AuthorityIndex, + application: BlockReference, + ) { + let now = Instant::now(); + if self + .payload_last_served + .get(&peer) + .is_some_and(|(_, last)| { + now.saturating_duration_since(*last) < SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 + }) + { + self.emit(ShadowServiceEventV1::Input { + kind: "application_payload_request", + outcome: "rate_limited", + }); + return; + } + let Some(payload) = self + .authorized_applications + .get(&application) + .and_then(|state| state.verified_payload.clone()) + else { + self.emit(ShadowServiceEventV1::Input { + kind: "application_payload_request", + outcome: "not_found", + }); + return; + }; + self.payload_last_served.insert(peer, (application, now)); + self.emit(ShadowServiceEventV1::Network { + recipient: peer, + message: NetworkMessage::RbcDagApplicationPayloadResponse( + RbcDagApplicationPayloadResponse { + application, + transaction_data: payload, + }, + ), + }); + } + + fn handle_application_payload_response( + &mut self, + peer: AuthorityIndex, + response: RbcDagApplicationPayloadResponse, + ) -> Result<(), ShadowServiceErrorV1> { + let application = response.application; + let Some(state) = self.authorized_applications.get(&application) else { + // Verification and network delivery run outside the actor. A + // solicited response can therefore arrive after the bounded + // application window evicted its state. Core either already saw + // the payload or later recovery will reauthorize the header; the + // delayed bytes must not invalidate an otherwise healthy run. + self.emit(ShadowServiceEventV1::Input { + kind: "application_payload_response", + outcome: "stale_ignored", + }); + return Ok(()); + }; + if state.verified_payload.is_some() { + return Ok(()); + } + if !state.request_last_attempt.contains_key(&peer) { + return Err( + ShadowServiceErrorV1::ApplicationPayloadResponseFromUnexpectedPeer { + peer, + application, + }, + ); + } + validate_application_payload_size(&response.transaction_data)?; + let state = self + .authorized_applications + .get_mut(&application) + .expect("authorized application cannot disappear during validation"); + let duplicate = application_payloads_equal( + state.observed_payload.as_deref(), + Some(response.transaction_data.as_ref()), + ); + merge_application_payload( + &mut state.observed_payload, + Some(response.transaction_data), + application, + )?; + state.observed_payload_at = Some(Instant::now()); + if !duplicate { + state.emitted_observed_payload = false; + } + self.emit_authorized_application_if_new(application); + Ok(()) + } + + fn handle_verified_application_payload( + &mut self, + application: BlockReference, + payload: Arc, + ) -> Result<(), ShadowServiceErrorV1> { + let Some(state) = self.authorized_applications.get_mut(&application) else { + // Core verification/materialization is deliberately offloaded + // from this actor. Its completion may therefore trail the bounded + // authorized-application retention window. The payload already + // reached Core; it must not turn normal bounded eviction into a + // protocol rejection or invalidate the benchmark. + self.emit(ShadowServiceEventV1::Input { + kind: "verified_application_payload", + outcome: "stale_ignored", + }); + return Ok(()); + }; + merge_application_payload(&mut state.verified_payload, Some(payload), application)?; + state.observed_payload = None; + state.observed_payload_at = None; + state.request_last_attempt.clear(); + self.emit(ShadowServiceEventV1::Input { + kind: "verified_application_payload", + outcome: "cached", + }); + Ok(()) + } + + fn reconcile_local_applications(&mut self) { + let desired = std::mem::take(&mut *self.desired_local_applications.lock()); + if !desired.is_empty() { + self.awaiting_application_submission = false; + } + for local in desired.into_values() { + self.enqueue_local(local); + } + } + + fn reconcile_verified_application_payloads(&mut self) { + let desired = std::mem::take(&mut *self.desired_verified_application_payloads.lock()); + for (application, payload) in desired { + if let Err(error) = self.handle_verified_application_payload(application, payload) { + self.reject(None, error); + } + } + } + + fn validity_backed_high_water( + &self, + hints: &BTreeMap, + ) -> Option { + let mut candidate_rounds = hints.values().copied().collect::>(); + candidate_rounds.sort_unstable(); + candidate_rounds.dedup(); + candidate_rounds.into_iter().rev().find(|round| { + let stake = hints + .iter() + .filter(|(_, high_water)| **high_water >= *round) + .filter_map(|(authority, _)| self.committee.committee().get_stake(*authority)) + .fold(0, u64::saturating_add); + self.committee.committee().is_valid(stake) + }) + } + + fn observe_catch_up_hint( + &mut self, + peer: AuthorityIndex, + round: RoundNumber, + outside_retention: bool, + ) { + if !self.mode.is_autonomous() { + return; + } + self.catch_up_hint_high_water + .entry(peer) + .and_modify(|high_water| *high_water = (*high_water).max(round)) + .or_insert(round); + if outside_retention { + self.far_future_hint_high_water + .entry(peer) + .and_modify(|high_water| *high_water = (*high_water).max(round)) + .or_insert(round); + } + let high_water = self.validity_backed_high_water(&self.catch_up_hint_high_water); + let far_high_water = self.validity_backed_high_water(&self.far_future_hint_high_water); + // Ordinary authenticated lookahead inside the 64-round retention + // window is the healthy proactive pipeline, not evidence that exact + // catch-up is needed. Starting a 64-slot repair episode for those + // observations made healthy nodes continuously request data they had + // already buffered. Activate pipelined catch-up only after validity + // stake reports a round outside normal retention. + let activated = !self.sync_catch_up && far_high_water.is_some(); + let limited = !self.sync_catch_up_limit_future && far_high_water.is_some(); + if far_high_water.is_some() { + let high_water = high_water.expect("far-future hints are also catch-up hints"); + self.sync_catch_up_target = Some( + self.sync_catch_up_target + .map_or(high_water, |target| target.max(high_water)), + ); + self.sync_catch_up = true; + } + if activated { + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_catch_up", + outcome: "validity_outside_retention_hints", + }); + } + if limited { + self.sync_catch_up = true; + self.sync_catch_up_limit_future = true; + } + } + + fn observe_ingress_catch_up_hint( + &mut self, + peer: AuthorityIndex, + canonical_carrier: &[u8], + disposition: ShadowIngressDispositionV1, + open_round_before: RoundNumber, + ) { + let Ok((_, hint_round, _)) = self.core.candidate_slot(canonical_carrier) else { + return; + }; + // Count deltas are not a sound lookahead signal: the same ingress can + // advance the clock, promote older buffered slots, and leave the total + // flat. Classify against the pre-ingress open round instead. + let authenticated_future = disposition == ShadowIngressDispositionV1::Authenticated + && hint_round > open_round_before.saturating_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1); + let future_ignored = disposition == ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer; + if !future_ignored && !authenticated_future { + return; + } + self.observe_catch_up_hint( + peer, + hint_round, + hint_round > open_round_before.saturating_add(EXECUTABLE_MODEL_BUFFER_WINDOW_V1), + ); + } + + /// Highest physical carrier round reported by validity stake while still + /// retained by the normal future window. A carrier at round `target` + /// proves that a live producer may already have opened `target + 1`, so + /// local catch-up is complete only after our open round passes it. + fn retained_future_target(&self) -> Option { + if !self.mode.is_autonomous() || self.sync_catch_up { + return None; + } + let target = self.validity_backed_high_water(&self.catch_up_hint_high_water)?; + (self.core.local_carrier_round() <= target).then_some(target) + } + + fn reconcile_topology(&mut self) { + let desired = self.desired_topology.lock().clone(); + let mut newly_connected = Vec::new(); + for (peer, state) in &desired { + if self.observed_topology.get(peer) == Some(state) { + continue; + } + self.recovery_last_attempt + .retain(|(_, holder), _| holder != peer); + self.sync_last_attempt + .retain(|(_, author), _| author != peer); + self.sync_last_served + .retain(|requester, _| requester != peer); + self.payload_last_served.remove(peer); + for application in self.authorized_applications.values_mut() { + application.request_last_attempt.remove(peer); + } + if state.0 { + self.connected.insert(*peer); + newly_connected.push(*peer); + } else { self.connected.remove(peer); + self.catch_up_hint_high_water.remove(peer); + self.far_future_hint_high_water.remove(peer); } } self.observed_topology = desired; @@ -1210,7 +2845,10 @@ impl ShadowServiceStateV1 { // Autonomous history is synchronized one exact slot at a // time. Replaying the entire retained run on every reconnect // would create an unbounded burst as heartbeats accumulate. - self.flush_carrier_sync_requests(self.core.local_carrier_round() > 1); + // The first physical carrier may have been fixed before the + // network connection existed, so round one needs the same + // immediate exact-slot repair as every later reconnect. + self.flush_carrier_sync_requests(true); } else { let retransmissions = self.core.retransmissions(); for peer in newly_connected { @@ -1220,6 +2858,7 @@ impl ShadowServiceStateV1 { } } self.flush_recovery_requests(); + self.flush_application_payload_requests(); } } @@ -1273,20 +2912,42 @@ impl ShadowServiceStateV1 { } fn broadcast(&self, envelope: &ShadowOutboundEnvelopeV1) { + self.broadcast_with_application_payload(envelope, None); + } + + fn broadcast_with_application_payload( + &self, + envelope: &ShadowOutboundEnvelopeV1, + application_payload: Option>, + ) { for recipient in 0..self.committee_size { let recipient = recipient as AuthorityIndex; if recipient != self.own_authority { - self.send_envelope(recipient, envelope); + self.send_envelope_with_application_payload( + recipient, + envelope, + application_payload.clone(), + ); } } } fn send_envelope(&self, recipient: AuthorityIndex, envelope: &ShadowOutboundEnvelopeV1) { + self.send_envelope_with_application_payload(recipient, envelope, None); + } + + fn send_envelope_with_application_payload( + &self, + recipient: AuthorityIndex, + envelope: &ShadowOutboundEnvelopeV1, + application_payload: Option>, + ) { self.emit(ShadowServiceEventV1::Network { recipient, message: NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { canonical_carrier: envelope.canonical_carrier_wire().to_vec(), authentication_sidecar: envelope.authentication_sidecar().to_vec(), + application_payload, }), }); } @@ -1305,21 +2966,17 @@ impl ShadowServiceStateV1 { } fn process_effects(&mut self, effects: Vec) { - let carrier_round_advanced = effects - .iter() - .any(|effect| matches!(effect, ModelEffect::CarrierRoundAdvanced(_))); let newly_delivered = effects .iter() .filter_map(|effect| match effect { - ModelEffect::Delivered(reference) => Some(*reference), + ModelEffect::Delivered(reference) | ModelEffect::DeliveryPromised(reference) => { + Some(*reference) + } ModelEffect::NeedCarrier { .. } | ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => None, }) .collect::>(); - if carrier_round_advanced { - self.sync_catch_up = std::mem::take(&mut self.sync_used_in_open_round); - } for effect in effects { match effect { ModelEffect::NeedCarrier { target, holders } => { @@ -1330,7 +2987,7 @@ impl ShadowServiceStateV1 { }), ); } - ModelEffect::Delivered(reference) => { + ModelEffect::Delivered(reference) | ModelEffect::DeliveryPromised(reference) => { self.pending_recovery.remove(&reference); } ModelEffect::PrefixAdvanced { .. } => {} @@ -1346,6 +3003,7 @@ impl ShadowServiceStateV1 { self.report_new_shadow_deliveries(&newly_delivered); self.report_projection_progress(); self.flush_carrier_sync_requests(false); + self.refresh_consensus_pacemaker(); self.emit_clock_state(); } @@ -1398,40 +3056,259 @@ impl ShadowServiceStateV1 { self.emit(ShadowServiceEventV1::LeaderDecided(decision)); } for delta in self.core.drain_committed_frontiers() { + if let Some(metrics) = &self.metrics { + let (total_ns, samples, max_ns) = latency_since_header_creation( + delta.applications.iter(), + current_timestamp_ns(), + ); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, + total_ns, + samples, + max_ns, + ); + metrics.starfish_rbc_dag_frontier_generated(); + } self.emit(ShadowServiceEventV1::FrontierCommitted(delta)); } } - fn try_create_autonomous_carrier(&mut self, allow_no_vote: bool) { - if !self.mode.is_autonomous() || !self.core.can_create_carrier() { - self.emit_clock_state(); - return; + fn refresh_consensus_pacemaker(&mut self) { + let _ = self.consensus_fallback_allowed_at(Instant::now()); + } + + fn consensus_fallback_allowed_at(&mut self, now: Instant) -> bool { + if !self.clock_active { + return false; } - let creation_time_ns = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - .try_into() - .unwrap_or(TimestampNs::MAX); - let before = self.core.wal_counts(); - let application_round = self.pending_local.keys().next().copied(); - let application = application_round.and_then(|round| self.pending_local.remove(&round)); - let result = match &application { - Some(application) => self.core.create_local_application_carrier( - application.application_header.clone(), - creation_time_ns, - allow_no_vote, - ), - None => self + let Some(leader_timeout) = self.mode.heartbeat_interval() else { + return false; + }; + let slot = self.core.next_local_consensus_round(); + let a1_ready = slot == 1 + || self .core - .create_local_control_heartbeat(creation_time_ns, allow_no_vote), + .has_projected_consensus_quorum(slot.saturating_sub(1)); + let c3_ready = self.core.has_projected_consensus_quorum(slot); + if self.consensus_pacemaker.slot != slot { + self.cancel_consensus_timeout_deadline(); + } + let fallback_allowed = self.consensus_pacemaker.fallback_allowed( + slot, + a1_ready, + c3_ready, + leader_timeout, + now, + ); + if fallback_allowed { + // C3 or an elapsed C2 already authorizes this exact slot. The + // normal-carrier permit may defer creation, but it will re-check + // the same monotonic evidence and does not need another timeout. + self.cancel_consensus_timeout_deadline(); + } else if let Some(armed_at) = self.consensus_pacemaker.c2_armed_at { + let deadline = armed_at.checked_add(leader_timeout).unwrap_or(now); + self.schedule_consensus_timeout_deadline(slot, deadline); + } else { + self.cancel_consensus_timeout_deadline(); + } + fallback_allowed + } + + fn cancel_consensus_timeout_deadline(&mut self) { + if self.consensus_timeout_deadline.take().is_some() { + self.consensus_timeout_generation = self.consensus_timeout_generation.wrapping_add(1); + self.consensus_timeout_deadline_tx.send_replace(None); + } + } + + fn schedule_consensus_timeout_deadline(&mut self, slot: RoundNumber, deadline: Instant) { + if self + .consensus_timeout_deadline + .is_some_and(|scheduled| scheduled.slot == slot && scheduled.deadline == deadline) + { + return; + } + self.consensus_timeout_generation = self.consensus_timeout_generation.wrapping_add(1); + let scheduled = ConsensusTimeoutDeadlineV1 { + generation: self.consensus_timeout_generation, + slot, + deadline, + }; + self.consensus_timeout_deadline = Some(scheduled); + self.consensus_timeout_deadline_tx + .send_replace(Some(scheduled)); + } + + fn observe_consensus_timeout_deadline(&mut self, generation: u64, slot: RoundNumber) { + let Some(scheduled) = self.consensus_timeout_deadline else { + return; + }; + if scheduled.generation != generation || scheduled.slot != slot { + return; + } + if Instant::now() < scheduled.deadline { + // Fail closed if an internal wake is ever observed early. Notify + // the persistent task so it continues waiting for the exact + // actor-owned deadline. + self.consensus_timeout_deadline_tx + .send_replace(Some(scheduled)); + return; + } + self.consensus_timeout_deadline = None; + self.consensus_timeout_deadline_tx.send_replace(None); + if self.core.next_local_consensus_round() != slot + || !self.consensus_pacemaker.observe_timeout(slot) + { + return; + } + self.awaiting_application_submission = false; + } + + fn cancel_normal_carrier_deadline(&mut self) { + if self.normal_carrier_deadline.take().is_some() { + self.normal_carrier_generation = self.normal_carrier_generation.wrapping_add(1); + self.normal_carrier_deadline_tx.send_replace(None); + } + } + + fn record_carrier_created(&mut self, now: Instant) { + self.normal_carrier_requested = false; + self.cancel_normal_carrier_deadline(); + self.normal_carrier_next_allowed_at = Some( + now.checked_add(self.normal_carrier_min_spacing) + .unwrap_or(now), + ); + } + + fn schedule_normal_carrier_deadline(&mut self, deadline: Instant) { + self.normal_carrier_requested = true; + if self + .normal_carrier_deadline + .is_some_and(|scheduled| scheduled.deadline == deadline) + { + return; + } + self.normal_carrier_generation = self.normal_carrier_generation.wrapping_add(1); + let scheduled = NormalCarrierDeadlineV1 { + generation: self.normal_carrier_generation, + deadline, + }; + self.normal_carrier_deadline = Some(scheduled); + self.normal_carrier_deadline_tx + .send_replace(Some(scheduled)); + } + + /// Request one normally paced carrier. All application, phase, heartbeat, + /// and C1/C2/C3 creation paths share this permit. Only validity-backed + /// exact/retained repair may call the unpaced primitive directly. + fn try_create_autonomous_carrier(&mut self) { + if !self.mode.is_autonomous() || !self.clock_active { + self.emit_clock_state(); + return; + } + let now = Instant::now(); + if let Some(deadline) = self.normal_carrier_next_allowed_at { + if now < deadline { + self.schedule_normal_carrier_deadline(deadline); + return; + } + } + self.normal_carrier_requested = false; + self.cancel_normal_carrier_deadline(); + let _ = self.try_create_autonomous_carrier_now(); + } + + fn observe_normal_carrier_deadline(&mut self, generation: u64) { + let Some(scheduled) = self.normal_carrier_deadline else { + return; + }; + if scheduled.generation != generation { + return; + } + self.normal_carrier_deadline = None; + self.normal_carrier_deadline_tx.send_replace(None); + if self.normal_carrier_requested { + self.try_create_autonomous_carrier(); + } + } + + /// C2 and C3 are creation triggers, not merely permission for the next + /// fixed heartbeat. Attempt at most one carrier outside `process_effects` + /// so reducing a locally created carrier cannot recurse into creation. + fn drive_consensus_fallback(&mut self) { + if !self.clock_active { + return; + } + let fallback_allowed = self.consensus_fallback_allowed_at(Instant::now()); + if !fallback_allowed + || !self.mode.is_autonomous() + || !self.core.can_create_carrier() + || (self.awaiting_application_submission && self.pending_local.is_empty()) + || self.fatal + { + return; + } + self.try_create_autonomous_carrier(); + } + + fn try_create_autonomous_carrier_now(&mut self) -> bool { + if !self.mode.is_autonomous() || !self.clock_active { + self.emit_clock_state(); + return false; + } + let allow_no_vote = self.consensus_fallback_allowed_at(Instant::now()); + if !self.core.can_create_carrier() { + self.emit_clock_state(); + return false; + } + let creation_time_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .try_into() + .unwrap_or(TimestampNs::MAX); + let before = self.core.wal_counts(); + let application_round = self.pending_local.keys().next().copied(); + let application = application_round.and_then(|round| self.pending_local.remove(&round)); + let result = match &application { + Some(application) => self.core.create_local_application_carrier( + application.application_header.clone(), + creation_time_ns, + allow_no_vote, + ), + None => self + .core + .create_local_control_heartbeat(creation_time_ns, allow_no_vote), }; match result { Ok((envelope, effects)) => { - if let Some(application) = application { + self.record_carrier_created(Instant::now()); + let initial_application_payload = application + .as_ref() + .and_then(|application| application.application_payload.clone()); + let assigned_application = application + .as_ref() + .filter(|application| application.acknowledge_assignment) + .map(|application| application.application_header.reference()); + if let Some(application) = &application { + if let Some(metrics) = &self.metrics { + let latency_ns = + creation_time_ns.saturating_sub(application.creation_time_ns); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, + latency_ns, + 1, + latency_ns, + ); + } self.assigned_applications .insert(application.application_header.reference()); } + // The application left the pending queue atomically with the + // successful carrier transition. Publish that state before + // releasing the producer gate or fan-out so observers cannot + // retain a stale nonzero current-depth gauge. + self.record_pipeline_state(); self.emit(ShadowServiceEventV1::Input { kind: if application_round.is_some() { "application_carrier" @@ -1441,8 +3318,25 @@ impl ShadowServiceStateV1 { outcome: "accepted", }); self.report_wal_delta(before); - self.broadcast(&envelope); + if let Some(reference) = assigned_application { + self.awaiting_application_submission = true; + self.emit(ShadowServiceEventV1::ApplicationAssigned(reference)); + } + if let Some(application) = application { + if let Err(error) = self.authorize_application( + envelope.reference(), + application.application_header, + application.application_payload, + true, + None, + ShadowApplicationAuthorizationBasisV1::LocallyFixed, + ) { + self.reject(None, error); + } + } + self.broadcast_with_application_payload(&envelope, initial_application_payload); self.process_effects(effects); + true } Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { if let Some(application) = application { @@ -1456,12 +3350,14 @@ impl ShadowServiceStateV1 { outcome: "waiting_for_quorum", }); self.emit_clock_state(); + false } Err(error) => { if let Some(application) = application { self.pending_local.insert(application.round, application); } self.mark_fatal(error); + false } } } @@ -1472,9 +3368,39 @@ impl ShadowServiceStateV1 { /// so a node behind a continuously advancing committee could never close /// the gap. Healthy rounds still remain paced exclusively by the timer. fn drive_autonomous_catch_up(&mut self) { - while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { + if !self.clock_active { + return; + } + for _ in 0..carrier_sync_pipeline_depth(self.committee_size) { + if !self.sync_catch_up || !self.core.can_create_carrier() || self.fatal { + break; + } + let round_before = self.core.local_carrier_round(); + let _ = self.try_create_autonomous_carrier_now(); + if self.core.local_carrier_round() == round_before { + break; + } + } + } + + /// Drain a validity-backed in-window tail without opening an exact repair + /// episode. Every iteration independently requires the exact predecessor + /// quorum and advances at most one physical round. Bound one actor turn so + /// its carrier fan-out remains within the same 64-message credit as exact + /// synchronization (`depth * (n - 1) <= 64`). + fn drive_retained_future_catch_up(&mut self) { + if !self.clock_active || self.fatal { + return; + } + let Some(target) = self.retained_future_target() else { + return; + }; + for _ in 0..carrier_sync_pipeline_depth(self.committee_size) { + if self.core.local_carrier_round() > target || !self.core.local_parent_quorum_ready() { + break; + } let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(true); + let _ = self.try_create_autonomous_carrier_now(); if self.core.local_carrier_round() == round_before { break; } @@ -1502,6 +3428,12 @@ impl ShadowServiceStateV1 { ); return; } + if let Some(payload) = &local.application_payload { + if let Err(error) = validate_application_payload_size(payload) { + self.reject(None, error); + return; + } + } if self.mode.is_autonomous() { let application_reference = local.application_header.reference(); if self.assigned_applications.contains(&application_reference) { @@ -1511,8 +3443,18 @@ impl ShadowServiceStateV1 { }); return; } - if let Some(existing) = self.pending_local.get(&local.round) { - if existing == &local { + if let Some(existing) = self.pending_local.get_mut(&local.round) { + if existing.same_application(&local) { + let result = merge_application_payload( + &mut existing.application_payload, + local.application_payload, + application_reference, + ); + existing.acknowledge_assignment |= local.acknowledge_assignment; + if let Err(error) = result { + self.reject(None, error); + return; + } self.emit(ShadowServiceEventV1::Input { kind: "application", outcome: "duplicate", @@ -1530,6 +3472,10 @@ impl ShadowServiceStateV1 { kind: "application", outcome: "queued", }); + // Capture queue occupancy before an immediately available carrier + // slot drains it; the current gauge will return to zero while the + // high-water mark preserves short head-of-line bursts. + self.record_pipeline_state(); self.retry_pending_local(); return; } @@ -1541,8 +3487,18 @@ impl ShadowServiceStateV1 { }); return; } - if let Some(existing) = self.pending_local.get(&local.round) { - if existing == &local { + if let Some(existing) = self.pending_local.get_mut(&local.round) { + if existing.same_application(&local) { + let result = merge_application_payload( + &mut existing.application_payload, + local.application_payload, + existing.application_header.reference(), + ); + existing.acknowledge_assignment |= local.acknowledge_assignment; + if let Err(error) = result { + self.reject(None, error); + return; + } self.emit(ShadowServiceEventV1::Input { kind: "local", outcome: "duplicate", @@ -1577,7 +3533,7 @@ impl ShadowServiceStateV1 { && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(false); + self.try_create_autonomous_carrier(); if self.core.local_carrier_round() == round_before { break; } @@ -1630,6 +3586,9 @@ impl ShadowServiceStateV1 { } fn flush_recovery_requests(&mut self) { + if self.mode.is_autonomous() && !self.clock_active { + return; + } let now = Instant::now(); let mut requests = Vec::new(); for (reference, holders) in &self.pending_recovery { @@ -1657,7 +3616,7 @@ impl ShadowServiceStateV1 { } fn flush_carrier_sync_requests(&mut self, force: bool) { - if !self.mode.is_autonomous() { + if !self.mode.is_autonomous() || !self.clock_active { return; } let round = self.core.local_carrier_round(); @@ -1665,13 +3624,26 @@ impl ShadowServiceStateV1 { if round != self.sync_round { self.sync_round = round; self.sync_round_opened_at = now; - self.sync_last_attempt.clear(); } - self.sync_last_attempt.retain(|(author, attempt_round), _| { - *attempt_round == round + self.sync_last_attempt.retain(|(attempt_round, author), _| { + *attempt_round >= round && self.connected.contains(author) - && self.core.admitted_reference(*author, round).is_none() + && self + .core + .authenticated_reference(*author, *attempt_round) + .is_none() }); + // A complete retained predecessor quorum can advance locally without + // network repair. In particular, suppress an already-expired ordinary + // grace timer while the current actor turn is about to drain that + // tail. A missing quorum still falls through to exact current-slot + // repair after the normal grace interval. + if !force + && self.retained_future_target().is_some() + && self.core.local_parent_quorum_ready() + { + return; + } if !force && !self.sync_catch_up && now.saturating_duration_since(self.sync_round_opened_at) @@ -1679,26 +3651,43 @@ impl ShadowServiceStateV1 { { return; } - let requests = self - .connected - .iter() - .copied() - .filter(|author| self.core.admitted_reference(*author, round).is_none()) - .filter(|author| { - self.sync_last_attempt - .get(&(*author, round)) - .is_none_or(|last| { - now.saturating_duration_since(*last) - >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 - }) - }) - .collect::>(); - for author in requests { - self.sync_last_attempt.insert((author, round), now); + let target = self + .sync_catch_up + .then_some(self.sync_catch_up_target) + .flatten(); + let candidates = + carrier_sync_pipeline_slots(round, target, self.committee_size, &self.connected); + for (request_round, author) in candidates { + if self + .core + .authenticated_reference(author, request_round) + .is_some() + { + continue; + } + let slot = (request_round, author); + let should_send = match self.sync_last_attempt.get(&slot) { + Some(last) => { + now.saturating_duration_since(*last) >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 + } + None => self.sync_last_attempt.len() < SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, + }; + if !should_send { + continue; + } + self.sync_last_attempt.insert(slot, now); + #[cfg(test)] + { + self.sync_max_outstanding = + self.sync_max_outstanding.max(self.sync_last_attempt.len()); + } self.emit(ShadowServiceEventV1::Network { recipient: author, message: NetworkMessage::RbcDagShadowCarrierSyncRequest( - RbcDagShadowCarrierSyncRequest { author, round }, + RbcDagShadowCarrierSyncRequest { + author, + round: request_round, + }, ), }); self.emit(ShadowServiceEventV1::Input { @@ -1706,6 +3695,31 @@ impl ShadowServiceStateV1 { outcome: "sent", }); } + self.finish_catch_up_if_drained(); + } + + fn finish_catch_up_if_drained(&mut self) { + if !self.sync_catch_up { + return; + } + let Some(target) = self.sync_catch_up_target else { + return; + }; + if self.core.local_carrier_round() < target + || !self.sync_last_attempt.is_empty() + || !self.desired_carrier_sync_responses.lock().is_empty() + { + return; + } + self.sync_catch_up = false; + self.sync_catch_up_limit_future = false; + self.sync_catch_up_target = None; + self.catch_up_hint_high_water.clear(); + self.far_future_hint_high_water.clear(); + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_catch_up", + outcome: "target_reached", + }); } fn handle_carrier_sync_request( @@ -1713,6 +3727,13 @@ impl ShadowServiceStateV1 { peer: AuthorityIndex, request: RbcDagShadowCarrierSyncRequest, ) { + if !self.clock_active { + self.emit(ShadowServiceEventV1::Input { + kind: "carrier_sync_request", + outcome: "paused", + }); + return; + } if request.author != self.own_authority { self.reject( Some(peer), @@ -1731,16 +3752,21 @@ impl ShadowServiceStateV1 { return; } let now = Instant::now(); - if self.sync_last_served.get(&peer).is_some_and(|(_, last)| { - now.saturating_duration_since(*last) < SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 - }) { + let permitted = match self.sync_last_served.get_mut(&peer) { + Some(window) => window.permits(request.round, now), + None => { + self.sync_last_served + .insert(peer, CarrierSyncServeWindowV1::first(request.round, now)); + true + } + }; + if !permitted { self.emit(ShadowServiceEventV1::Input { kind: "carrier_sync_request", outcome: "rate_limited", }); return; } - self.sync_last_served.insert(peer, (request.round, now)); let Some(envelope) = self.core.local_outbound_envelope(request.round) else { self.emit(ShadowServiceEventV1::Input { kind: "carrier_sync_request", @@ -1770,7 +3796,7 @@ impl ShadowServiceStateV1 { peer: AuthorityIndex, response: RbcDagShadowCarrierSyncResponse, ) { - let expected = (response.author, response.round); + let expected = (response.round, response.author); if peer != response.author { self.reject( Some(peer), @@ -1804,7 +3830,7 @@ impl ShadowServiceStateV1 { if response.round < self.core.local_carrier_round() || self .core - .admitted_reference(response.author, response.round) + .authenticated_reference(response.author, response.round) .is_some() { self.sync_last_attempt.remove(&expected); @@ -1825,6 +3851,9 @@ impl ShadowServiceStateV1 { return; } let before = self.core.wal_counts(); + // Exact requested responses always use the normal 64-round + // authenticated window, including while unsolicited proactive + // traffic is narrowed in far catch-up mode. match self.core.receive_or_retain_from_peer( &response.canonical_carrier, &response.authentication_sidecar, @@ -1835,25 +3864,41 @@ impl ShadowServiceStateV1 { ShadowIngressDispositionV1::Authenticated => "authenticated", ShadowIngressDispositionV1::CandidateRetained => "retained_unauthenticated", ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => "ignored", + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => "future_ignored", }; self.emit(ShadowServiceEventV1::Input { kind: "carrier_sync_response", outcome: outcome_label, }); + if let Err(error) = self.observe_carrier_application( + peer, + &response.canonical_carrier, + None, + outcome.disposition(), + ) { + self.reject(Some(peer), error); + } if outcome.disposition() == ShadowIngressDispositionV1::Authenticated && self .core - .admitted_reference(response.author, response.round) + .authenticated_reference(response.author, response.round) == Some(actual_reference) { - self.sync_used_in_open_round = true; + // Every fresh process repairs its carrier fixed before + // topology registration at round one. That alone is not + // lag evidence and must not turn healthy startup into a + // permanent exact-sync loop. Later authenticated repairs + // do prove that this clock missed live traffic. + if response.round > 1 { + self.observe_catch_up_hint(peer, response.round, false); + } } self.report_wal_delta(before); self.process_effects(outcome.effects().to_vec()); self.retry_pending_local(); if self .core - .admitted_reference(response.author, response.round) + .authenticated_reference(response.author, response.round) .is_some() { self.sync_last_attempt.remove(&expected); @@ -1882,6 +3927,45 @@ impl ShadowServiceStateV1 { } } + /// Drain exact-slot repair ahead of the ordinary proactive-carrier FIFO. + /// A globally bounded set of exact `(round, author)` responses is retained + /// by the handle. This is the priority seam a lagging honest node needs to + /// consume pipelined repair while a live quorum continues producing future + /// carriers. + fn reconcile_carrier_sync_responses(&mut self) { + if !self.clock_active { + return; + } + loop { + #[cfg(test)] + { + self.sync_max_desired_responses = self + .sync_max_desired_responses + .max(self.desired_carrier_sync_responses.lock().len()); + } + let current_round = self.core.local_carrier_round(); + let next = { + let mut desired = self.desired_carrier_sync_responses.lock(); + let exact_slot = desired.iter().find_map(|(slot, (_, response))| { + (response.round == current_round).then_some(*slot) + }); + let slot = exact_slot.or_else(|| desired.keys().next().copied()); + slot.and_then(|slot| desired.remove(&slot)) + }; + let Some((peer, response)) = next else { + break; + }; + if let Err(error) = self.validate_peer(peer) { + self.reject(Some(peer), error); + continue; + } + self.handle_carrier_sync_response(peer, response); + if self.fatal { + break; + } + } + } + fn report_new_shadow_deliveries(&mut self, references: &[BlockReference]) { for reference in references { let identity = match self.core.delivery_identity(*reference) { @@ -1903,10 +3987,26 @@ impl ShadowServiceStateV1 { self.emit_comparison_backlog(); match self.core.delivered_application_header(*reference) { Ok(Some((carrier, header))) => { + if let Err(error) = + self.authorize_delivered_application(carrier, header.clone()) + { + self.reject(None, error); + return; + } if self .reported_application_deliveries .insert(header.reference()) { + if let Some(metrics) = &self.metrics { + let latency_ns = current_timestamp_ns() + .saturating_sub(header.meta_creation_time_ns()); + metrics.observe_starfish_rbc_dag_pipeline_latency_ns( + RBC_DAG_LATENCY_CREATION_TO_DELIVERY, + latency_ns, + 1, + latency_ns, + ); + } self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header, @@ -1978,6 +4078,7 @@ fn run_shadow_service( mut state: ShadowServiceStateV1, mut messages: mpsc::Receiver, open_report: ShadowOpenReportV1, + clock_starts_active: bool, ) { if open_report.replayed_batches() != 0 || open_report.discarded_tail_bytes() != 0 { state.emit(ShadowServiceEventV1::Recovered { @@ -1986,20 +4087,42 @@ fn run_shadow_service( }); } state.reconcile_topology(); + state.reconcile_local_applications(); + state.reconcile_verified_application_payloads(); state.reconcile_direct_deliveries(); if !state.observe_external_invalidation() { - state.emit(ShadowServiceEventV1::Ready { - autonomous_clock: state.mode.is_autonomous(), - }); - state.emit_comparison_backlog(); - state.process_effects(open_report.recovery_effects().to_vec()); - state.retry_pending_local(); + if state.mode.is_autonomous() { + // Replay-derived authority and effects must cross the ordered + // event bridge before readiness can release an authoritative + // Core. Clock activation follows Ready in the active-by-default + // path and remains an explicit later message when coordinated. + state.emit_autonomous_recovery_and_ready(&open_report); + if clock_starts_active { + state.activate_clock(); + state.retry_pending_local(); + } + } else { + // Direct mirror mode is observational and preserves its existing + // Ready-first event contract for comparison consumers. + state.emit(ShadowServiceEventV1::Ready { + autonomous_clock: false, + }); + state.emit_recovered_authorized_applications(); + state.emit_comparison_backlog(); + state.process_effects(open_report.recovery_effects().to_vec()); + state.retry_pending_local(); + } + state.drive_consensus_fallback(); if state.mode.is_autonomous() { state.emit_clock_state(); } } while !state.fatal { + state.reconcile_carrier_sync_responses(); + if state.fatal { + break; + } let Some(message) = messages.blocking_recv() else { break; }; @@ -2034,8 +4157,17 @@ fn run_shadow_service( _ => {} } match message { - ShadowServiceMessageV1::LocalCarrier(local) => { - state.enqueue_local(local); + ShadowServiceMessageV1::ActivateClock(reply) => { + state.activate_clock(); + // A paused open may already hold a recovered or live local + // application. Release that event-driven work immediately + // after the ordered ClockActivated observation; control-only + // production still waits for the fresh heartbeat epoch. + state.retry_pending_local(); + let _ = reply.send(()); + } + ShadowServiceMessageV1::LocalApplicationsChanged => { + state.reconcile_local_applications(); } ShadowServiceMessageV1::Carrier { peer, envelope } => { if let Err(error) = state.validate_peer(peer) { @@ -2043,12 +4175,21 @@ fn run_shadow_service( continue; } let before = state.core.wal_counts(); - match state.core.receive_or_retain_from_peer( + let open_round_before = state.core.local_carrier_round(); + let catchup_limited = state.sync_catch_up_limit_future; + match state.core.receive_or_retain_from_peer_with_future_window( &envelope.canonical_carrier, &envelope.authentication_sidecar, peer, + if catchup_limited { + 0 + } else { + EXECUTABLE_MODEL_BUFFER_WINDOW_V1 + }, ) { Ok(outcome) => { + let future_ignored = outcome.disposition() + == ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer; let outcome_label = match outcome.disposition() { ShadowIngressDispositionV1::Authenticated => "authenticated", ShadowIngressDispositionV1::CandidateRetained => { @@ -2057,14 +4198,39 @@ fn run_shadow_service( ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { "ignored" } + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer + if catchup_limited => + { + "catchup_future_ignored" + } + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => { + "future_ignored" + } }; state.emit(ShadowServiceEventV1::Input { kind: "carrier", outcome: outcome_label, }); + state.observe_ingress_catch_up_hint( + peer, + &envelope.canonical_carrier, + outcome.disposition(), + open_round_before, + ); + if let Err(error) = state.observe_carrier_application( + peer, + &envelope.canonical_carrier, + envelope.application_payload, + outcome.disposition(), + ) { + state.reject(Some(peer), error); + } state.report_wal_delta(before); state.process_effects(outcome.effects().to_vec()); state.retry_pending_local(); + if future_ignored { + state.flush_carrier_sync_requests(true); + } if outcome.disposition() == ShadowIngressDispositionV1::CandidateRetained { state.reject( Some(peer), @@ -2149,6 +4315,14 @@ fn run_shadow_service( kind: "recovery", outcome: "accepted", }); + if let Err(error) = state.observe_carrier_application( + peer, + &response.canonical_carrier, + None, + ShadowIngressDispositionV1::CandidateRetained, + ) { + state.reject(Some(peer), error); + } state.report_wal_delta(before); state.process_effects(effects); state.retry_pending_local(); @@ -2173,33 +4347,86 @@ fn run_shadow_service( } state.handle_carrier_sync_request(peer, request); } - ShadowServiceMessageV1::CarrierSyncResponse { peer, response } => { + ShadowServiceMessageV1::CarrierSyncResponsesChanged => {} + ShadowServiceMessageV1::ApplicationPayloadRequest { peer, application } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + state.handle_application_payload_request(peer, application); + } + ShadowServiceMessageV1::ApplicationPayloadResponse { peer, response } => { if let Err(error) = state.validate_peer(peer) { state.reject(Some(peer), error); continue; } - state.handle_carrier_sync_response(peer, response); + if let Err(error) = state.handle_application_payload_response(peer, response) { + state.reject(Some(peer), error); + } } + ShadowServiceMessageV1::VerifiedApplicationPayloadsChanged => {} ShadowServiceMessageV1::DirectDeliveriesChanged => { state.reconcile_direct_deliveries(); } - ShadowServiceMessageV1::TopologyChanged => state.reconcile_topology(), + ShadowServiceMessageV1::TopologyChanged => {} ShadowServiceMessageV1::RetryRecovery => { - state.reconcile_topology(); + state.awaiting_application_submission = false; + state.reconcile_local_applications(); state.reconcile_pending_recovery(); state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); + state.flush_application_payload_requests(); + } + ShadowServiceMessageV1::HeartbeatTick => { + state.awaiting_application_submission = false; + state.try_create_autonomous_carrier(); + } + ShadowServiceMessageV1::NormalCarrierDeadline { generation } => { + state.observe_normal_carrier_deadline(generation); + } + ShadowServiceMessageV1::ConsensusTimeoutDeadline { generation, slot } => { + state.observe_consensus_timeout_deadline(generation, slot); } - ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(true), ShadowServiceMessageV1::DataAvailabilityChanged => { state.reconcile_data_availability(); } + #[cfg(test)] + ShadowServiceMessageV1::InspectRbcProgress(reply) => { + let progress = ( + state.core.optimistic_promise_count(), + state + .core + .certified_delivery_count() + .expect("test progress inspection requires unambiguous deliveries"), + ); + let _ = reply.send(progress); + } + #[cfg(test)] + ShadowServiceMessageV1::InspectCarrierSync(reply) => { + let desired_responses = state.desired_carrier_sync_responses.lock().len(); + state.sync_max_desired_responses = + state.sync_max_desired_responses.max(desired_responses); + let _ = reply.send(CarrierSyncInspectionV1 { + open_round: state.core.local_carrier_round(), + target: state.sync_catch_up_target, + outstanding: state.sync_last_attempt.len(), + desired_responses, + max_outstanding: state.sync_max_outstanding, + max_desired_responses: state.sync_max_desired_responses, + }); + } ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); + state.reconcile_local_applications(); + state.reconcile_carrier_sync_responses(); + state.reconcile_verified_application_payloads(); state.reconcile_direct_deliveries(); + state.drive_consensus_fallback(); state.drive_autonomous_catch_up(); + state.drive_retained_future_catch_up(); state.flush_carrier_sync_requests(false); + state.flush_application_payload_requests(); } let events = state.events.clone(); if let Err(error) = state.core.shutdown() { @@ -2217,6 +4444,29 @@ fn delivery_slot(identity: &ShadowDeliveryIdentityV1) -> ShadowDeliverySlotV1 { } } +fn current_timestamp_ns() -> TimestampNs { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .try_into() + .unwrap_or(TimestampNs::MAX) +} + +fn latency_since_header_creation<'a>( + headers: impl Iterator, + now_ns: TimestampNs, +) -> (u64, u64, u64) { + headers.fold((0u64, 0u64, 0u64), |(total, samples, maximum), header| { + let latency = now_ns.saturating_sub(header.meta_creation_time_ns()); + ( + total.saturating_add(latency), + samples.saturating_add(1), + maximum.max(latency), + ) + }) +} + fn authentication_sidecar_size(scheme: BlockAuthenticationScheme, committee_size: usize) -> usize { const SIDECAR_HEADER_SIZE: usize = 3; SIDECAR_HEADER_SIZE @@ -2263,6 +4513,55 @@ fn validate_wire_size( } } +fn validate_application_payload_size( + payload: &TransactionData, +) -> Result<(), ShadowServiceErrorV1> { + let actual = bincode::serialized_size(payload) + .map_err(|error| ShadowServiceErrorV1::ApplicationPayloadSerialization(error.to_string()))? + .try_into() + .unwrap_or(usize::MAX); + validate_wire_size( + "application payload", + actual, + SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1, + ) +} + +fn application_payloads_equal( + left: Option<&TransactionData>, + right: Option<&TransactionData>, +) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.transactions() == right.transactions(), + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } +} + +fn merge_application_payload( + retained: &mut Option>, + incoming: Option>, + application: BlockReference, +) -> Result<(), ShadowServiceErrorV1> { + let Some(incoming) = incoming else { + return Ok(()); + }; + match retained { + Some(existing) + if !application_payloads_equal(Some(existing.as_ref()), Some(incoming.as_ref())) => + { + Err(ShadowServiceErrorV1::ConflictingApplicationPayload( + application, + )) + } + Some(_) => Ok(()), + None => { + *retained = Some(incoming); + Ok(()) + } + } +} + fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { matches!( error, @@ -2272,8 +4571,16 @@ fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { #[cfg(test)] mod tests { - use std::{sync::Arc, time::Duration}; + use std::{ + collections::VecDeque, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering as AtomicOrdering}, + }, + time::Duration, + }; + use prometheus::Registry; use tempfile::TempDir; use tokio::time::timeout; @@ -2281,34 +4588,864 @@ mod tests { use crate::{ committee::Committee, crypto::{TransactionsCommitment, mac_keyrings_for_test}, + encoder::{Encoder, ShardEncoder}, starfish_rbc_dag::{ - CandidateCarrierV1, CarrierAuthorizerV1, CarrierHeaderV1Args, RbcDagProtocolInstanceId, - RbcPhaseStatementV1, carrier_genesis_reference, + CandidateCarrierV1, CarrierAuthorizerV1, CarrierHeaderV1Args, LeaderChoiceV1, + RbcDagProtocolInstanceId, RbcPhaseStatementV1, carrier_genesis_reference, }, - types::{BlockDigest, VerifiedBlock}, + types::{BaseTransaction, BlockDigest, Transaction, VerifiedBlock}, }; const N: usize = 4; const EVENT_TIMEOUT: Duration = Duration::from_secs(5); - struct Harness { - _directory: TempDir, - committee: RbcDagCommitteeContextV1, - context: RbcDagContextV1, - keyrings: Vec>, - paths: Vec, + #[test] + fn carrier_sync_limiter_is_order_independent_and_throttles_replay() { + let start = Instant::now(); + let first = SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1 as RoundNumber; + let mut limiter = CarrierSyncServeWindowV1::first(first, start); + for round in (1..first).rev() { + assert!(limiter.permits(round, start)); + } + let next = first.saturating_add(1); + assert!(!limiter.permits(next, start)); + assert!(!limiter.permits(1, start)); + + let later = start + SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1; + assert!(limiter.permits(next, later)); + assert!(!limiter.permits(next, later)); + assert!(limiter.permits(1, later)); } - impl Harness { - fn new() -> Self { - Self::new_with_n(N) - } + #[test] + fn ten_validator_sync_window_is_round_major_and_bounded_to_sixty_three_slots() { + let connected = (1..10) + .map(|authority| authority as AuthorityIndex) + .collect(); + let slots = carrier_sync_pipeline_slots(40, Some(100), 10, &connected); + + assert_eq!(carrier_sync_pipeline_depth(10), 7); + assert_eq!(slots.len(), 63); + assert_eq!(slots.first(), Some(&(40, 1))); + assert_eq!(slots.last(), Some(&(46, 9))); + assert!(slots.windows(2).all(|pair| pair[0] < pair[1])); + assert!(!slots.iter().any(|(round, _)| *round >= 47)); + assert!(slots.len() <= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1); + + let moderate = carrier_sync_pipeline_slots(40, None, 10, &connected); + assert_eq!(moderate.len(), 9); + assert!(moderate.iter().all(|(round, _)| *round == 40)); + } - fn new_with_n(n: usize) -> Self { - let committee = Committee::new_test(vec![1; n]); - let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), + #[test] + fn exact_sync_responses_coalesce_by_slot_without_overwrite() { + let (sender, _receiver) = mpsc::channel(1); + let desired = Arc::new(Mutex::new(BTreeMap::new())); + let handle = StarfishRbcDagShadowServiceHandleV1 { + sender, + max_sidecar_size: 3 + N * MAC_TAG_SIZE, + own_authority: 0, + committee_size: N, + input_capacity: 1, + mode: ShadowServiceModeV1::AutonomousClock { + heartbeat_interval: Duration::from_secs(1), + }, + desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), + desired_carrier_sync_responses: Arc::clone(&desired), + desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), + desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), + invalidated_by_overload: Arc::new(Mutex::new(None)), + }; + let response = |author, round, marker| RbcDagShadowCarrierSyncResponse { + author, + round, + canonical_carrier: vec![marker], + authentication_sidecar: Vec::new(), + }; + + let first = response(1, 7, 0xA1); + let later = response(1, 8, 0xA2); + handle.carrier_sync_response(1, first.clone()).unwrap(); + handle.carrier_sync_response(1, later.clone()).unwrap(); + handle.carrier_sync_response(1, first.clone()).unwrap(); + assert_eq!(desired.lock().len(), 2); + assert_eq!(desired.lock().get(&(7, 1)).unwrap().1, first); + assert_eq!(desired.lock().get(&(8, 1)).unwrap().1, later); + + assert!(matches!( + handle.carrier_sync_response(1, response(1, 7, 0xFF)), + Err(ShadowServiceErrorV1::UnexpectedSyncResponse { + author: 1, + round: 7, + }) + )); + assert_eq!( + desired.lock().get(&(7, 1)).unwrap().1.canonical_carrier, + vec![0xA1] + ); + + 'fill: for round in 1..=RoundNumber::MAX { + for author in 1..N as AuthorityIndex { + if desired.lock().len() == SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 { + break 'fill; + } + if desired.lock().contains_key(&(round, author)) { + continue; + } + handle + .carrier_sync_response(author, response(author, round, author as u8)) + .unwrap(); + } + } + assert_eq!(desired.lock().len(), SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1); + assert!(matches!( + handle.carrier_sync_response(1, response(1, 10_000, 0xCC)), + Err(ShadowServiceErrorV1::CarrierSyncResponseCapacity { + capacity: SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, + }) + )); + } + + #[tokio::test] + async fn catch_up_target_is_the_monotone_validity_stake_high_water() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (state, _events, _message_rx, _message_tx) = + standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); + tokio::task::spawn_blocking(move || { + let mut state = state; + state.observe_catch_up_hint(1, 100, true); + assert_eq!(state.sync_catch_up_target, None); + state.observe_catch_up_hint(2, 70, true); + assert_eq!(state.sync_catch_up_target, Some(70)); + assert!(state.sync_catch_up); + assert!(state.sync_catch_up_limit_future); + + state.observe_catch_up_hint(3, 90, true); + assert_eq!(state.sync_catch_up_target, Some(90)); + state.observe_catch_up_hint(2, 110, true); + assert_eq!(state.sync_catch_up_target, Some(100)); + state.observe_catch_up_hint(1, 80, true); + assert_eq!(state.sync_catch_up_target, Some(100)); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn in_window_future_hints_do_not_start_exact_catch_up() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (state, _events, _message_rx, _message_tx) = + standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); + tokio::task::spawn_blocking(move || { + let mut state = state; + state.observe_catch_up_hint(1, 40, false); + assert_eq!(state.retained_future_target(), None); + // Two remote hints carry validity stake in the four-node test, + // but both remain inside normal authenticated retention. + state.observe_catch_up_hint(2, 40, false); + assert_eq!( + state.validity_backed_high_water(&state.catch_up_hint_high_water), + Some(40) + ); + assert_eq!(state.retained_future_target(), Some(40)); + assert!(!state.sync_catch_up); + assert!(!state.sync_catch_up_limit_future); + assert_eq!(state.sync_catch_up_target, None); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + } + + fn producer_carrier( + committee: &RbcDagCommitteeContextV1, + chains: &[Vec], + author: AuthorityIndex, + round: RoundNumber, + ) -> CandidateCarrierV1 { + let previous = |authority: AuthorityIndex| { + if round == 1 { + carrier_genesis_reference(authority) + } else { + chains[authority as usize][round as usize - 2] + } + }; + let mut parent_stake = committee + .committee() + .get_stake(author) + .expect("producer authority belongs to the committee"); + let mut weak_parents = Vec::new(); + for parent_authority in committee.committee().authorities() { + // Authority zero is the lagger. A live quorum of the other + // producers must remain able to extend without naming its stale + // physical chain. + if parent_authority == 0 || parent_authority == author { + continue; + } + weak_parents.push(previous(parent_authority)); + parent_stake = parent_stake.saturating_add( + committee + .committee() + .get_stake(parent_authority) + .expect("parent authority belongs to the committee"), + ); + if parent_stake >= committee.committee().quorum_threshold() { + break; + } + } + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: round, + own_prev: previous(author), + weak_parents, + transactions_commitment: TransactionsCommitment::default(), + application_header: None, + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: round.into(), + }, + committee, + ) + .unwrap() + } + + fn feed_producer_round( + state: &mut ShadowServiceStateV1, + harness: &Harness, + chains: &mut [Vec], + round: RoundNumber, + ) { + let open_before = state.core.local_carrier_round(); + for author in 1..state.committee_size as AuthorityIndex { + let candidate = producer_carrier(&harness.committee, chains, author, round); + let reference = candidate.reference(); + let envelope = harness.envelope(&candidate, author); + let outcome = state + .core + .receive_or_retain_from_peer( + &envelope.canonical_carrier, + &envelope.authentication_sidecar, + author, + ) + .unwrap(); + assert_eq!( + outcome.disposition(), + ShadowIngressDispositionV1::Authenticated + ); + state.observe_ingress_catch_up_hint( + author, + &envelope.canonical_carrier, + outcome.disposition(), + open_before, + ); + state.process_effects(outcome.effects().to_vec()); + chains[author as usize].push(reference); + } + } + + async fn assert_retained_future_tail_closes_without_exact_sync(n: usize, initial_tail: u32) { + let harness = Harness::new_with_n(n); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (mut state, mut events, _messages, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + let sync_requests = Arc::new(AtomicUsize::new(0)); + let sync_requests_observed = Arc::clone(&sync_requests); + let event_drain = tokio::spawn(async move { + while let Some(event) = events.recv().await { + if matches!( + event, + ShadowServiceEventV1::Network { + message: NetworkMessage::RbcDagShadowCarrierSyncRequest(_), + .. + } + ) { + sync_requests_observed.fetch_add(1, AtomicOrdering::Relaxed); + } + } + }); + + tokio::task::spawn_blocking(move || { + let mut chains = vec![Vec::new(); n]; + state.connected = (1..n as AuthorityIndex).collect(); + state.clock_active = false; + for round in 1..=initial_tail { + feed_producer_round(&mut state, &harness, &mut chains, round); + } + assert_eq!(state.core.local_carrier_round(), 1); + assert_eq!(state.retained_future_target(), Some(initial_tail)); + + state.activate_clock(); + state.sync_round_opened_at = Instant::now() - Duration::from_secs(60 * 60 * 3); + let depth = carrier_sync_pipeline_depth(n) as RoundNumber; + let mut healthy_round = initial_tail; + for _ in 0..initial_tail.saturating_add(4) { + // Keep the healthy quorum producing while the lagger consumes + // its retained tail. + healthy_round = healthy_round.saturating_add(1); + feed_producer_round(&mut state, &harness, &mut chains, healthy_round); + let before = state.core.local_carrier_round(); + state.drive_retained_future_catch_up(); + let after = state.core.local_carrier_round(); + assert!(after.saturating_sub(before) <= depth); + state.flush_carrier_sync_requests(false); + if healthy_round.saturating_add(1).saturating_sub(after) <= 2 { + break; + } + } + let gap = healthy_round + .saturating_add(1) + .saturating_sub(state.core.local_carrier_round()); + assert!(gap <= 2, "retained local catch-up plateaued with gap {gap}"); + assert!(!state.sync_catch_up); + assert!(!state.sync_catch_up_limit_future); + + let buffered_capacity = EXECUTABLE_MODEL_BUFFER_WINDOW_V1 + .saturating_sub(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) + as usize + * n.saturating_sub(1); + assert!(state.core.buffered_authenticated_carrier_count() <= buffered_capacity); + + // Stop producers and prove the target-carrier off-by-one: fixing + // the hinted carrier round opens its successor. + if let Some(final_target) = state.retained_future_target() { + for _ in 0..initial_tail.saturating_add(4) { + state.drive_retained_future_catch_up(); + if state.core.local_carrier_round() > final_target { + break; + } + } + assert_eq!(state.core.local_carrier_round(), final_target + 1); + } + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + event_drain.await.unwrap(); + assert_eq!(sync_requests.load(AtomicOrdering::Relaxed), 0); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn four_node_retained_gap_seventeen_closes_without_exact_sync() { + assert_retained_future_tail_closes_without_exact_sync(4, 17).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn ten_node_retained_gap_thirty_two_closes_without_exact_sync() { + assert_retained_future_tail_closes_without_exact_sync(10, 32).await; + } + + #[tokio::test] + async fn requested_future_response_uses_normal_window_during_far_catch_up() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (state, _events, _message_rx, _message_tx) = + standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); + let round = 10; + let previous = |authority: AuthorityIndex| BlockReference { + authority, + round: round - 1, + digest: BlockDigest::from([0x70 + authority as u8; 32]), + }; + let candidate = CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author: 1, + carrier_round: round, + own_prev: previous(1), + weak_parents: [0, 2].into_iter().map(previous).collect(), + transactions_commitment: TransactionsCommitment::default(), + application_header: None, + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: round.into(), + }, + &harness.committee, + ) + .unwrap(); + let reference = candidate.reference(); + let envelope = harness.envelope(&candidate, 1); + tokio::task::spawn_blocking(move || { + let mut state = state; + state.sync_catch_up_limit_future = true; + state.sync_last_attempt.insert((round, 1), Instant::now()); + state.handle_carrier_sync_response( + 1, + RbcDagShadowCarrierSyncResponse { + author: 1, + round, + canonical_carrier: envelope.canonical_carrier, + authentication_sidecar: envelope.authentication_sidecar, + }, + ); + + assert_eq!(state.core.local_carrier_round(), 1); + assert_eq!( + state.core.authenticated_reference(1, round), + Some(reference) + ); + assert!(!state.sync_last_attempt.contains_key(&(round, 1))); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + } + + #[test] + fn consensus_pacemaker_fixed_grid_does_not_inherit_c2_for_a_new_slot() { + let timeout = Duration::from_millis(100); + let origin = Instant::now(); + let mut pacemaker = ConsensusPacemakerV1::new(2); + + assert!(!pacemaker.fallback_allowed(2, true, false, timeout, origin)); + assert!(pacemaker.fallback_allowed(2, true, false, timeout, origin + timeout)); + assert!( + !pacemaker.fallback_allowed(3, true, false, timeout, origin + timeout), + "a fixed-grid tick at the instant slot 3 opens must reset C2" + ); + assert!(!pacemaker.fallback_allowed( + 3, + true, + false, + timeout, + origin + timeout + timeout - Duration::from_nanos(1), + )); + assert!(pacemaker.fallback_allowed(3, true, false, timeout, origin + timeout + timeout,)); + } + + #[test] + fn consensus_pacemaker_c2_starts_only_when_a1_becomes_ready() { + let timeout = Duration::from_millis(100); + let origin = Instant::now(); + let mut pacemaker = ConsensusPacemakerV1::new(4); + + assert!(!pacemaker.fallback_allowed(4, false, false, timeout, origin)); + assert!(!pacemaker.fallback_allowed( + 4, + false, + false, + timeout, + origin + timeout.saturating_mul(10), + )); + let a1_at = origin + timeout.saturating_mul(10); + assert!(!pacemaker.fallback_allowed(4, true, false, timeout, a1_at)); + assert!(!pacemaker.fallback_allowed( + 4, + true, + false, + timeout, + a1_at + timeout - Duration::from_nanos(1), + )); + assert!(pacemaker.fallback_allowed(4, true, false, timeout, a1_at + timeout)); + } + + #[test] + fn consensus_pacemaker_c3_authorizes_immediate_slot_bound_catch_up() { + let timeout = Duration::from_secs(60); + let origin = Instant::now(); + let mut pacemaker = ConsensusPacemakerV1::new(7); + + assert!(pacemaker.fallback_allowed(7, false, true, timeout, origin)); + assert!( + !pacemaker.fallback_allowed(8, false, false, timeout, origin), + "C3 evidence for slot 7 must not leak into slot 8" + ); + } + + #[tokio::test] + async fn service_c3_emits_a_control_carrier_without_waiting_for_the_fixed_grid() { + let harness = Harness::new(); + let (mut core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let reference = |authority, consensus_round, marker| { + ConsensusVertexReference::new( + BlockReference { + authority, + round: 100 + consensus_round, + digest: BlockDigest::from([marker; 32]), + }, + consensus_round, + ) + }; + + let round_one_leader = reference(1, 1, 0x11); + core.inject_projected_consensus_for_test( + round_one_leader, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: 0, + leader_round: 0, + }, + ); + for (author, choice) in [ + ( + 0, + LeaderChoiceV1::Vote { + leader: round_one_leader, + }, + ), + ( + 1, + LeaderChoiceV1::Vote { + leader: round_one_leader, + }, + ), + ( + 3, + LeaderChoiceV1::NoVote { + leader_author: 1, + leader_round: 1, + }, + ), + ] { + core.inject_projected_consensus_for_test( + reference(author, 2, 0x20 + author as u8), + vec![round_one_leader], + choice, + ); + } + for author in [1, 2, 3] { + core.inject_projected_consensus_for_test( + reference(author, 3, 0x30 + author as u8), + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author: 2, + leader_round: 2, + }, + ); + } + core.set_next_local_consensus_round_for_test(3); + assert!(core.has_projected_consensus_quorum(3)); + + let (state, mut event_rx, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + + let state = tokio::task::spawn_blocking(move || { + let mut state = state; + state.drive_consensus_fallback(); + state + }) + .await + .unwrap(); + assert_eq!(state.core.next_local_consensus_round(), 4); + let mut emitted = None; + while let Ok(event) = event_rx.try_recv() { + if let ShadowServiceEventV1::Network { + message: NetworkMessage::RbcDagShadowCarrier(carrier), + .. + } = event + { + emitted = Some(carrier); + break; + } + } + let emitted = emitted.expect("C3 must schedule a carrier immediately"); + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &emitted.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + let vertex = candidate + .header() + .consensus_vertex() + .expect("C3 carrier must contain the lagging logical slot"); + assert_eq!(vertex.consensus_round(), 3); + assert_eq!( + vertex.leader_choice(), + LeaderChoiceV1::NoVote { + leader_author: 2, + leader_round: 2, + } + ); + state.core.shutdown().unwrap(); + } + + #[tokio::test] + async fn service_c2_deadline_message_creates_without_waiting_for_the_next_grid_tick() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let leader_timeout = Duration::from_millis(20); + let (mut state, mut event_rx, mut message_rx, _message_tx) = + standalone_autonomous_state(core, harness.committee.clone(), leader_timeout); + + state.refresh_consensus_pacemaker(); + assert_eq!(state.core.local_carrier_round(), 1); + let (generation, slot) = match timeout(Duration::from_secs(1), message_rx.recv()) + .await + .expect("slot-bound C2 deadline was not scheduled") + { + Some(ShadowServiceMessageV1::ConsensusTimeoutDeadline { generation, slot }) => { + (generation, slot) + } + other => panic!( + "unexpected deadline message: {:?}", + other.map(|message| message.kind()) + ), + }; + assert_eq!(slot, 1); + state.observe_consensus_timeout_deadline(generation, slot); + assert!(state.consensus_pacemaker.c2_timed_out); + + let state = tokio::task::spawn_blocking(move || { + state.drive_consensus_fallback(); + state + }) + .await + .unwrap(); + assert!(!state.core.can_create_carrier()); + assert_eq!(state.core.next_local_consensus_round(), 2); + let mut emitted = false; + while let Ok(event) = event_rx.try_recv() { + emitted |= matches!( + event, + ShadowServiceEventV1::Network { + message: NetworkMessage::RbcDagShadowCarrier(_), + .. + } + ); + } + assert!(emitted, "C2 deadline must schedule a carrier immediately"); + state.core.shutdown().unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn consensus_timeout_deadline_coalesces_and_rejects_stale_wakes() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (mut state, _event_rx, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + + state.refresh_consensus_pacemaker(); + let scheduled = state + .consensus_timeout_deadline + .expect("C2 readiness did not arm its persistent deadline"); + assert_eq!(scheduled.slot, 1); + + // Repeated ingress/maintenance observations for one logical slot + // share one generation and one physical timer. + for _ in 0..8 { + state.refresh_consensus_pacemaker(); + assert_eq!(state.consensus_timeout_deadline, Some(scheduled)); + assert_eq!(state.consensus_timeout_generation, scheduled.generation); + } + + state.observe_consensus_timeout_deadline( + scheduled.generation.wrapping_sub(1), + scheduled.slot, + ); + assert_eq!(state.consensus_timeout_deadline, Some(scheduled)); + assert!(!state.consensus_pacemaker.c2_timed_out); + + // Make the unit-state deadline due without sleeping for an hour. The + // current generation is consumed once; a duplicate queued wake is a + // no-op and cannot authorize a different slot. + let due = Instant::now() + .checked_sub(Duration::from_millis(1)) + .unwrap_or_else(Instant::now); + let due_scheduled = ConsensusTimeoutDeadlineV1 { + deadline: due, + ..scheduled + }; + state.consensus_timeout_deadline = Some(due_scheduled); + state + .consensus_timeout_deadline_tx + .send_replace(Some(due_scheduled)); + state.observe_consensus_timeout_deadline(due_scheduled.generation, due_scheduled.slot); + assert_eq!(state.consensus_timeout_deadline, None); + assert!(state.consensus_pacemaker.c2_timed_out); + state.observe_consensus_timeout_deadline(due_scheduled.generation, due_scheduled.slot); + assert_eq!(state.consensus_timeout_deadline, None); + + // Replacing the actor-owned target leaves exactly one desired + // deadline. Advancing the logical slot cancels it, and its already + // queued generation remains harmless. + let replacement_deadline = Instant::now() + Duration::from_secs(60 * 60); + state.schedule_consensus_timeout_deadline(1, replacement_deadline); + let replacement = state.consensus_timeout_deadline.unwrap(); + assert_ne!(replacement.generation, scheduled.generation); + state.core.set_next_local_consensus_round_for_test(2); + state.refresh_consensus_pacemaker(); + assert_eq!(state.consensus_pacemaker.slot, 2); + assert_eq!(state.consensus_timeout_deadline, None); + state.observe_consensus_timeout_deadline(replacement.generation, replacement.slot); + assert_eq!(state.consensus_timeout_deadline, None); + + state.core.shutdown().unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn normal_carrier_deadline_coalesces_and_rejects_stale_generations() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (mut state, mut event_rx, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + + tokio::task::spawn_blocking(move || { + state.normal_carrier_min_spacing = Duration::from_secs(60 * 60); + state.try_create_autonomous_carrier(); + assert_eq!(state.core.local_carrier_round(), 1); + assert!(!state.core.can_create_carrier()); + + // Every normal trigger while the permit is closed must share one + // generation and one deadline, regardless of its source. + state.try_create_autonomous_carrier(); + let scheduled = state + .normal_carrier_deadline + .expect("closed permit did not schedule its persistent wake"); + assert!(state.normal_carrier_requested); + for _ in 0..8 { + state.try_create_autonomous_carrier(); + assert_eq!(state.normal_carrier_deadline, Some(scheduled)); + assert_eq!(state.normal_carrier_generation, scheduled.generation); + } + + let stale_generation = scheduled.generation.wrapping_sub(1); + state.observe_normal_carrier_deadline(stale_generation); + assert_eq!(state.normal_carrier_deadline, Some(scheduled)); + assert!(state.normal_carrier_requested); + + // Admit the exact predecessor quorum so the current wake can fix + // one, and only one, successor carrier. Move the deterministic + // unit-state deadline to the past instead of sleeping for an hour. + let mut chains = vec![Vec::new(); N]; + feed_producer_round(&mut state, &harness, &mut chains, 1); + assert_eq!(state.core.local_carrier_round(), 2); + assert!(state.core.can_create_carrier()); + let due = Instant::now() + .checked_sub(Duration::from_millis(1)) + .unwrap_or_else(Instant::now); + let due_scheduled = NormalCarrierDeadlineV1 { + generation: scheduled.generation, + deadline: due, + }; + state.normal_carrier_next_allowed_at = Some(due); + state.normal_carrier_deadline = Some(due_scheduled); + state + .normal_carrier_deadline_tx + .send_replace(Some(due_scheduled)); + + state.observe_normal_carrier_deadline(due_scheduled.generation); + assert_eq!(state.core.local_carrier_round(), 2); + assert!(!state.core.can_create_carrier()); + assert!(!state.normal_carrier_requested); + assert_eq!(state.normal_carrier_deadline, None); + + // A duplicate queued wake for the generation just consumed is a + // no-op and cannot spend another physical slot. + state.observe_normal_carrier_deadline(due_scheduled.generation); + assert_eq!(state.core.local_carrier_round(), 2); + assert!(!state.core.can_create_carrier()); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + + let proactive_carriers = std::iter::from_fn(|| event_rx.try_recv().ok()) + .filter(|event| { + matches!( + event, + ShadowServiceEventV1::Network { + message: NetworkMessage::RbcDagShadowCarrier(_), + .. + } + ) + }) + .count(); + assert_eq!(proactive_carriers, 2 * (N - 1)); + } + + struct Harness { + _directory: TempDir, + committee: RbcDagCommitteeContextV1, + context: RbcDagContextV1, + keyrings: Vec>, + paths: Vec, + } + + impl Harness { + fn new() -> Self { + Self::new_with_n(N) + } + + fn new_with_n(n: usize) -> Self { + let committee = Committee::new_test(vec![1; n]); + let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), &committee, BlockAuthenticationScheme::MacVector, ); @@ -2396,6 +5533,31 @@ mod tests { .unwrap() } + fn start_autonomous_paused_with_interval( + &self, + authority: AuthorityIndex, + heartbeat_interval: Duration, + ) -> ( + StarfishRbcDagShadowServiceHandleV1, + mpsc::Receiver, + JoinHandle<()>, + ) { + start_starfish_rbc_dag_shadow_service_with_mode_v1( + &self.paths[authority as usize], + self.committee.clone(), + authority, + self.context, + ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), + Vec::new(), + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + ShadowWalSyncPolicyV1::EveryBatch, + None, + None, + false, + ) + .unwrap() + } + fn envelope( &self, candidate: &CandidateCarrierV1, @@ -2415,10 +5577,102 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: candidate.canonical_wire_bytes().unwrap(), authentication_sidecar: authentication.canonical_wire_bytes(), + application_payload: None, } } } + fn standalone_autonomous_state( + core: StarfishRbcDagShadowV1, + committee: RbcDagCommitteeContextV1, + leader_timeout: Duration, + ) -> ( + ShadowServiceStateV1, + mpsc::Receiver, + mpsc::Receiver, + mpsc::Sender, + ) { + let slot = core.next_local_consensus_round(); + let committee_size = committee.committee().len(); + let (event_tx, event_rx) = mpsc::channel(128); + let (message_tx, message_rx) = mpsc::channel(8); + let (clock_activation_tx, _clock_activation_rx) = watch::channel(true); + let (normal_carrier_deadline_tx, _normal_carrier_deadline_rx) = watch::channel(None); + let (consensus_timeout_deadline_tx, consensus_timeout_deadline_rx) = watch::channel(None); + spawn_consensus_timeout_deadline_task( + consensus_timeout_deadline_rx, + message_tx.downgrade(), + ); + let state = ShadowServiceStateV1 { + core, + committee, + mode: ShadowServiceModeV1::AutonomousClock { + heartbeat_interval: leader_timeout, + }, + clock_active: true, + clock_activation_tx, + wal_sync_policy: ShadowWalSyncPolicyV1::EveryBatch, + metrics: None, + own_authority: 0, + committee_size, + events: event_tx, + connected: BTreeSet::new(), + catch_up_hint_high_water: BTreeMap::new(), + far_future_hint_high_water: BTreeMap::new(), + desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), + desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), + desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), + desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), + observed_topology: BTreeMap::new(), + invalidated_by_overload: Arc::new(Mutex::new(None)), + pending_local: BTreeMap::new(), + assigned_applications: BTreeSet::new(), + pending_data_availability: BTreeSet::new(), + pending_recovery: BTreeMap::new(), + recovery_last_attempt: BTreeMap::new(), + sync_last_attempt: BTreeMap::new(), + sync_last_served: BTreeMap::new(), + authorized_applications: BTreeMap::new(), + quarantined_application_payloads: BTreeMap::new(), + payload_last_served: BTreeMap::new(), + sync_round: 1, + sync_round_opened_at: Instant::now(), + sync_catch_up: false, + sync_catch_up_limit_future: false, + sync_catch_up_target: None, + sync_max_outstanding: 0, + sync_max_desired_responses: 0, + awaiting_application_submission: false, + consensus_pacemaker: ConsensusPacemakerV1::new(slot), + // Existing state-level tests invoke creation synchronously and do + // not run the service deadline task. Keep their historical + // immediate behavior unless a pacer test overrides this field. + normal_carrier_min_spacing: Duration::ZERO, + normal_carrier_next_allowed_at: None, + normal_carrier_requested: false, + normal_carrier_generation: 0, + normal_carrier_deadline: None, + normal_carrier_deadline_tx, + consensus_timeout_generation: 0, + consensus_timeout_deadline: None, + consensus_timeout_deadline_tx, + retry_notification_pending: Arc::new(AtomicBool::new(false)), + heartbeat_notification_pending: Arc::new(AtomicBool::new(false)), + direct_deliveries: BTreeSet::new(), + reported_shadow_deliveries: BTreeSet::new(), + reported_application_deliveries: BTreeSet::new(), + recovered_shadow_deliveries: BTreeSet::new(), + comparison_backlog: ShadowComparisonBacklogV1::new(BTreeSet::new()), + reported_matches: BTreeSet::new(), + reported_mismatches: BTreeSet::new(), + reported_conflicts: BTreeSet::new(), + fatal: false, + }; + (state, event_rx, message_rx, message_tx) + } + async fn next_event(events: &mut mpsc::Receiver) -> ShadowServiceEventV1 { timeout(EVENT_TIMEOUT, events.recv()) .await @@ -2433,57 +5687,289 @@ mod tests { ShadowServiceEventV1::Rejected { error, .. } => { panic!("shadow startup failed: {error}") } - _ => {} - } - } - } - - async fn wait_backlog( - events: &mut mpsc::Receiver, - expected: (usize, usize, RoundNumber), - ) { - loop { - match next_event(events).await { - ShadowServiceEventV1::ComparisonBacklog { - unpaired_direct, - unpaired_shadow, - max_round_lag, - } if (unpaired_direct, unpaired_shadow, max_round_lag) == expected => return, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("shadow service rejected input while waiting for backlog: {error}") + _ => {} + } + } + } + + #[tokio::test] + async fn recovered_control_frontier_precedes_ready_and_clock_activation() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (mut state, mut events, _messages, _message_tx) = + standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); + state.clock_active = false; + let delta = CommittedFrontierDeltaV1 { + output_sequence: 1, + anchor: ConsensusVertexReference::new(BlockReference::new_test(1, 10), 7), + frontier: vec![None; harness.committee.committee().len()], + carriers: Vec::new(), + applications: Vec::new(), + application_diagnostics: Vec::new(), + }; + let report = + ShadowOpenReportV1::with_recovered_committed_frontiers_for_test(vec![delta.clone()]); + + state = tokio::task::spawn_blocking(move || { + state.emit_autonomous_recovery_and_ready(&report); + state + }) + .await + .unwrap(); + assert!(matches!( + next_event(&mut events).await, + ShadowServiceEventV1::FrontierCommitted(actual) if actual == delta + )); + loop { + match next_event(&mut events).await { + ShadowServiceEventV1::Ready { + autonomous_clock: true, + } => break, + ShadowServiceEventV1::PendingRecovery(_) + | ShadowServiceEventV1::ClockState { .. } => {} + unexpected => panic!("unexpected recovery-prefix event: {unexpected:?}"), + } + } + assert!(events.try_recv().is_err()); + + state = tokio::task::spawn_blocking(move || { + state.activate_clock(); + state + }) + .await + .unwrap(); + assert!(matches!( + next_event(&mut events).await, + ShadowServiceEventV1::ClockActivated + )); + state.core.shutdown().unwrap(); + } + + async fn wait_backlog( + events: &mut mpsc::Receiver, + expected: (usize, usize, RoundNumber), + ) { + loop { + match next_event(events).await { + ShadowServiceEventV1::ComparisonBacklog { + unpaired_direct, + unpaired_shadow, + max_round_lag, + } if (unpaired_direct, unpaired_shadow, max_round_lag) == expected => return, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("shadow service rejected input while waiting for backlog: {error}") + } + _ => {} + } + } + } + + async fn next_carrier( + events: &mut mpsc::Receiver, + expected_recipient: AuthorityIndex, + ) -> RbcDagShadowCarrier { + loop { + if let ShadowServiceEventV1::Network { + recipient, + message: NetworkMessage::RbcDagShadowCarrier(envelope), + } = next_event(events).await + { + if recipient == expected_recipient { + return envelope; + } + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn pump_autonomous_until_round( + handles: &[StarfishRbcDagShadowServiceHandleV1], + events: &mut [mpsc::Receiver], + open_rounds: &mut [RoundNumber], + deliveries: &mut [usize], + application_deliveries: &mut [BTreeSet], + committed_frontiers: &mut [Vec], + sync_requests: &mut usize, + projected_vertices: &mut usize, + projected_decisions: &mut usize, + max_buffered_authenticated: &mut usize, + rejections: &mut Vec, + target_open_round: RoundNumber, + pump_timeout: Duration, + ) { + timeout(pump_timeout, async { + let mut pending_network = VecDeque::new(); + loop { + let mut progressed = false; + for sender in 0..events.len() { + // Route at most one event per node per pass. Draining one + // sender completely can fill a target actor while that + // target is blocked publishing into its own bounded event + // channel, creating a test-router cycle absent from the + // independent production bridges. + if let Ok(event) = events[sender].try_recv() { + progressed = true; + match event { + ShadowServiceEventV1::Network { recipient, message } => { + let recipient = recipient as usize; + let service_message = match message { + NetworkMessage::RbcDagShadowCarrier(envelope) => { + ShadowServiceMessageV1::Carrier { + peer: sender as AuthorityIndex, + envelope, + } + } + NetworkMessage::RbcDagShadowCarrierRequest(reference) => { + ShadowServiceMessageV1::CarrierRequest { + peer: sender as AuthorityIndex, + reference, + } + } + NetworkMessage::RbcDagShadowCarrierResponse(response) => { + ShadowServiceMessageV1::CarrierResponse { + peer: sender as AuthorityIndex, + response, + } + } + NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { + *sync_requests = sync_requests.saturating_add(1); + ShadowServiceMessageV1::CarrierSyncRequest { + peer: sender as AuthorityIndex, + request, + } + } + NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { + handles[recipient] + .carrier_sync_response( + sender as AuthorityIndex, + response, + ) + .unwrap(); + continue; + } + NetworkMessage::RbcDagApplicationPayloadRequest(application) => { + ShadowServiceMessageV1::ApplicationPayloadRequest { + peer: sender as AuthorityIndex, + application, + } + } + NetworkMessage::RbcDagApplicationPayloadResponse(response) => { + ShadowServiceMessageV1::ApplicationPayloadResponse { + peer: sender as AuthorityIndex, + response, + } + } + unexpected => panic!( + "autonomous shadow emitted unexpected network message: {unexpected:?}" + ), + }; + pending_network.push_back((recipient, service_message)); + } + ShadowServiceEventV1::ClockState { + open_round, + buffered_authenticated, + .. + } => { + open_rounds[sender] = open_rounds[sender].max(open_round); + *max_buffered_authenticated = + (*max_buffered_authenticated).max(buffered_authenticated); + } + ShadowServiceEventV1::Delivered(_) => { + deliveries[sender] = deliveries[sender].saturating_add(1); + } + ShadowServiceEventV1::EmbeddedApplicationDelivered { + header, + .. + } => { + application_deliveries[sender].insert(header.reference()); + } + ShadowServiceEventV1::AuthorizedApplicationObserved { + header, + payload, + .. + } => { + let reference = header.reference(); + if let Some(payload) = payload { + handles[sender] + .verified_application_payload(reference, payload) + .unwrap(); + } + // The production bridge emits availability + // only after the typed header (and optional + // payload) is concretely installed in Core. + handles[sender].application_data_available(reference).unwrap(); + } + ShadowServiceEventV1::VertexProjected(_) => { + *projected_vertices = projected_vertices.saturating_add(1); + } + ShadowServiceEventV1::LeaderDecided(_) => { + *projected_decisions = projected_decisions.saturating_add(1); + } + ShadowServiceEventV1::FrontierCommitted(delta) => { + committed_frontiers[sender].push(delta); + } + ShadowServiceEventV1::Rejected { error, .. } + if error.contains("unexpected shadow response") => + { + rejections.push(error); + } + ShadowServiceEventV1::Rejected { error, .. } => { + rejections.push(error.clone()); + panic!("autonomous shadow rejected valid test traffic: {error}") + } + _ => {} + } + } + } + let pending = pending_network.len(); + for _ in 0..pending { + let (recipient, message) = pending_network + .pop_front() + .expect("pending network length was captured"); + match handles[recipient].sender.try_send(message) { + Ok(()) => progressed = true, + Err(TrySendError::Full(message)) => { + pending_network.push_back((recipient, message)); + } + Err(TrySendError::Closed(_)) => { + panic!("autonomous shadow target actor stopped") + } + } + } + if open_rounds + .iter() + .all(|round| *round >= target_open_round) + && pending_network.is_empty() + { + return; } - _ => {} - } - } - } - - async fn next_carrier( - events: &mut mpsc::Receiver, - expected_recipient: AuthorityIndex, - ) -> RbcDagShadowCarrier { - loop { - if let ShadowServiceEventV1::Network { - recipient, - message: NetworkMessage::RbcDagShadowCarrier(envelope), - } = next_event(events).await - { - if recipient == expected_recipient { - return envelope; + if !progressed { + tokio::time::sleep(Duration::from_millis(1)).await; + } else { + tokio::task::yield_now().await; } } - } + }) + .await + .unwrap_or_else(|_| { + panic!( + "autonomous clock did not open round {target_open_round}; open={open_rounds:?}, sync_requests={sync_requests}" + ) + }); } - async fn pump_autonomous_until_round( + async fn pump_online_prefix_until_round( handles: &[StarfishRbcDagShadowServiceHandleV1], events: &mut [mpsc::Receiver], open_rounds: &mut [RoundNumber], - deliveries: &mut [usize], - application_deliveries: &mut [BTreeSet], - committed_frontiers: &mut [Vec], - sync_requests: &mut usize, - projected_vertices: &mut usize, - projected_decisions: &mut usize, + online: usize, target_open_round: RoundNumber, ) { timeout(EVENT_TIMEOUT, async { @@ -2493,7 +5979,9 @@ mod tests { while let Ok(event) = events[sender].try_recv() { progressed = true; match event { - ShadowServiceEventV1::Network { recipient, message } => { + ShadowServiceEventV1::Network { recipient, message } + if sender < online && (recipient as usize) < online => + { let recipient = recipient as usize; match message { NetworkMessage::RbcDagShadowCarrier(envelope) => handles @@ -2509,7 +5997,6 @@ mod tests { .carrier_response(sender as AuthorityIndex, response) .unwrap(), NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { - *sync_requests = sync_requests.saturating_add(1); handles[recipient] .carrier_sync_request( sender as AuthorityIndex, @@ -2525,35 +6012,44 @@ mod tests { ) .unwrap(); } + NetworkMessage::RbcDagApplicationPayloadRequest(application) => { + handles[recipient] + .application_payload_request( + sender as AuthorityIndex, + application, + ) + .unwrap(); + } + NetworkMessage::RbcDagApplicationPayloadResponse(response) => { + handles[recipient] + .application_payload_response( + sender as AuthorityIndex, + response, + ) + .unwrap(); + } unexpected => panic!( "autonomous shadow emitted unexpected network message: {unexpected:?}" ), } } + ShadowServiceEventV1::Network { .. } => {} ShadowServiceEventV1::ClockState { open_round, .. } => { open_rounds[sender] = open_rounds[sender].max(open_round); } - ShadowServiceEventV1::Delivered(_) => { - deliveries[sender] = deliveries[sender].saturating_add(1); - } - ShadowServiceEventV1::EmbeddedApplicationDelivered { + ShadowServiceEventV1::AuthorizedApplicationObserved { header, + payload, .. } => { - application_deliveries[sender].insert(header.reference()); - } - ShadowServiceEventV1::VertexProjected(_) => { - *projected_vertices = projected_vertices.saturating_add(1); - } - ShadowServiceEventV1::LeaderDecided(_) => { - *projected_decisions = projected_decisions.saturating_add(1); - } - ShadowServiceEventV1::FrontierCommitted(delta) => { - committed_frontiers[sender].push(delta); + let reference = header.reference(); + if let Some(payload) = payload { + handles[sender] + .verified_application_payload(reference, payload) + .unwrap(); + } + handles[sender].application_data_available(reference).unwrap(); } - ShadowServiceEventV1::Rejected { error, .. } - if error.contains("FutureCarrierOutsideBuffer") - || error.contains("unexpected shadow response") => {} ShadowServiceEventV1::Rejected { error, .. } => { panic!("autonomous shadow rejected valid test traffic: {error}") } @@ -2561,143 +6057,673 @@ mod tests { } } } - if open_rounds + if open_rounds[..online] .iter() .all(|round| *round >= target_open_round) { return; } - if !progressed { - tokio::time::sleep(Duration::from_millis(1)).await; - } else { - tokio::task::yield_now().await; + if !progressed { + tokio::time::sleep(Duration::from_millis(1)).await; + } else { + tokio::task::yield_now().await; + } + } + }) + .await + .unwrap_or_else(|_| panic!("online prefix did not open round {target_open_round}")); + } + + async fn stop( + handle: StarfishRbcDagShadowServiceHandleV1, + events: mpsc::Receiver, + task: JoinHandle<()>, + ) { + drop(events); + handle.shutdown().await.unwrap(); + task.await.unwrap(); + } + + async fn startup_rejection(mut events: mpsc::Receiver) -> String { + loop { + match next_event(&mut events).await { + ShadowServiceEventV1::Rejected { peer: None, error } => return error, + ShadowServiceEventV1::Ready { .. } => { + panic!("invalid shadow startup became ready") + } + _ => {} + } + } + } + + fn direct_header(author: AuthorityIndex, round: RoundNumber, marker: u8) -> RbcCanonicalHeader { + RbcCanonicalHeader::try_new( + author, + round, + (0..N) + .map(|authority| { + *VerifiedBlock::new_genesis(authority as AuthorityIndex).reference() + }) + .collect(), + Vec::new(), + u64::from(round) * 1_000 + u64::from(marker), + TransactionsCommitment::from_bytes([marker; 32]), + ) + .unwrap() + } + + fn application_header_and_payload( + author: AuthorityIndex, + round: RoundNumber, + marker: u8, + committee: &RbcDagCommitteeContextV1, + ) -> (RbcCanonicalHeader, Arc) { + let payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![marker; 64]), + )])); + let info_length = committee.committee().info_length(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded = encoder.encode_transactions( + payload.transactions(), + info_length, + committee.committee().len() - info_length, + ); + let commitment = + TransactionsCommitment::new_from_encoded_transactions(&encoded, author as usize).0; + let header = RbcCanonicalHeader::try_new( + author, + round, + committee + .committee() + .authorities() + .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) + .collect(), + Vec::new(), + u64::from(round) * 1_000 + u64::from(marker), + commitment, + ) + .unwrap(); + (header, payload) + } + + fn round_one_application_candidate( + author: AuthorityIndex, + application_header: RbcCanonicalHeader, + committee: &RbcDagCommitteeContextV1, + marker: u8, + ) -> CandidateCarrierV1 { + let weak_parents = committee + .committee() + .authorities() + .filter(|authority| *authority != author) + .take(2) + .map(carrier_genesis_reference) + .collect(); + CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author, + carrier_round: 1, + own_prev: carrier_genesis_reference(author), + weak_parents, + transactions_commitment: application_header.transactions_commitment(), + application_header: Some(application_header), + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: u64::from(marker), + }, + committee, + ) + .unwrap() + } + + #[tokio::test] + async fn far_future_application_is_ignored_before_authentication_or_payload_observation() { + let harness = Harness::new(); + let (header, payload) = application_header_and_payload(0, 1, 0xC0, &harness.committee); + let previous = |authority: AuthorityIndex| BlockReference { + authority, + round: 65, + digest: BlockDigest::from([0xC0 + authority as u8; 32]), + }; + let candidate = CandidateCarrierV1::try_new_with_committee( + CarrierHeaderV1Args { + author: 0, + carrier_round: 66, + own_prev: previous(0), + weak_parents: [1, 2].into_iter().map(previous).collect(), + transactions_commitment: header.transactions_commitment(), + application_header: Some(header), + data_acknowledgments: Vec::new(), + phase_batch: Vec::new(), + consensus_vertex: None, + creation_time_ns: 66, + }, + &harness.committee, + ) + .unwrap(); + let (receiver, mut events, task) = harness.start_autonomous(1); + wait_ready(&mut events).await; + receiver.peer_connected(0).unwrap(); + loop { + if let ShadowServiceEventV1::Network { + recipient: 0, + message: NetworkMessage::RbcDagShadowCarrierSyncRequest(request), + } = next_event(&mut events).await + { + assert_eq!((request.author, request.round), (0, 1)); + break; + } + } + tokio::time::sleep(SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1).await; + receiver + .carrier( + 0, + RbcDagShadowCarrier { + canonical_carrier: candidate.canonical_wire_bytes().unwrap(), + authentication_sidecar: vec![0xff], + application_payload: Some(payload), + }, + ) + .unwrap(); + + let mut ignored = false; + let mut repair_requested = false; + while !ignored || !repair_requested { + match next_event(&mut events).await { + ShadowServiceEventV1::Input { + kind: "carrier", + outcome: "future_ignored", + } => ignored = true, + ShadowServiceEventV1::Network { + recipient: 0, + message: NetworkMessage::RbcDagShadowCarrierSyncRequest(request), + } => { + assert_eq!((request.author, request.round), (0, 1)); + repair_requested = true; + } + ShadowServiceEventV1::AuthorizedApplicationObserved { .. } => { + panic!("far-future application was observed") + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("far-future application was rejected instead of ignored: {error}") + } + _ => {} + } + } + stop(receiver, events, task).await; + } + + #[tokio::test] + async fn initial_payload_fanout_and_authenticated_observation_are_exact() { + let harness = Harness::new(); + let (header, payload) = application_header_and_payload(0, 1, 0xC1, &harness.committee); + + let (author, mut author_events, author_task) = harness.start_autonomous(0); + wait_ready(&mut author_events).await; + author + .local_application(&header, Some(Arc::clone(&payload))) + .unwrap(); + let mut assigned = false; + let mut locally_authorized = false; + let initial = loop { + match next_event(&mut author_events).await { + ShadowServiceEventV1::ApplicationAssigned(reference) + if reference == header.reference() => + { + assigned = true; + } + ShadowServiceEventV1::AuthorizedApplicationObserved { + header: observed, + authorization_basis: ShadowApplicationAuthorizationBasisV1::LocallyFixed, + .. + } if observed == header => { + assert!( + assigned, + "producer gate must be released before materialization" + ); + locally_authorized = true; + } + ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrier(initial), + } => { + assert!(locally_authorized); + break initial; + } + _ => {} + } + }; + assert!(initial.application_payload.is_some()); + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &initial.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + assert_eq!( + candidate + .header() + .application_header() + .map(RbcCanonicalHeader::reference), + Some(header.reference()) + ); + + let (receiver, mut receiver_events, receiver_task) = harness.start_autonomous(1); + wait_ready(&mut receiver_events).await; + receiver.carrier(0, initial).unwrap(); + loop { + match next_event(&mut receiver_events).await { + ShadowServiceEventV1::AuthorizedApplicationObserved { + carrier, + header: observed, + payload: Some(observed_payload), + authorization_basis: + ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, + } => { + assert_eq!(carrier, candidate.reference()); + assert_eq!(observed, header); + assert!(application_payloads_equal( + Some(observed_payload.as_ref()), + Some(payload.as_ref()) + )); + break; + } + ShadowServiceEventV1::Delivered(_) => { + panic!("fresh receiver authentication should stage before delivery") + } + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("valid authenticated application was rejected: {error}") } + _ => {} } - }) - .await - .unwrap_or_else(|_| { - panic!( - "autonomous clock did not open round {target_open_round}; open={open_rounds:?}, sync_requests={sync_requests}" - ) - }); + } + stop(author, author_events, author_task).await; + stop(receiver, receiver_events, receiver_task).await; } - async fn pump_online_prefix_until_round( - handles: &[StarfishRbcDagShadowServiceHandleV1], - events: &mut [mpsc::Receiver], - open_rounds: &mut [RoundNumber], - online: usize, - target_open_round: RoundNumber, - ) { - timeout(EVENT_TIMEOUT, async { - loop { - let mut progressed = false; - for sender in 0..events.len() { - while let Ok(event) = events[sender].try_recv() { - progressed = true; - match event { - ShadowServiceEventV1::Network { recipient, message } - if sender < online && (recipient as usize) < online => - { - let recipient = recipient as usize; - match message { - NetworkMessage::RbcDagShadowCarrier(envelope) => handles - [recipient] - .carrier(sender as AuthorityIndex, envelope) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierRequest(reference) => handles - [recipient] - .carrier_request(sender as AuthorityIndex, reference) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierResponse(response) => handles - [recipient] - .carrier_response(sender as AuthorityIndex, response) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { - handles[recipient] - .carrier_sync_request( - sender as AuthorityIndex, - request, - ) - .unwrap(); - } - NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { - handles[recipient] - .carrier_sync_response( - sender as AuthorityIndex, - response, - ) - .unwrap(); - } - unexpected => panic!( - "autonomous shadow emitted unexpected network message: {unexpected:?}" - ), - } - } - ShadowServiceEventV1::Network { .. } => {} - ShadowServiceEventV1::ClockState { open_round, .. } => { - open_rounds[sender] = open_rounds[sender].max(open_round); - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("autonomous shadow rejected valid test traffic: {error}") - } - _ => {} - } - } + #[tokio::test] + async fn authorized_empty_application_never_requests_payload_across_retries() { + let harness = Harness::new(); + let header = RbcCanonicalHeader::try_new( + 0, + 1, + harness + .committee + .committee() + .authorities() + .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) + .collect(), + Vec::new(), + 1_001, + TransactionsCommitment::default(), + ) + .unwrap(); + let candidate = + round_one_application_candidate(0, header.clone(), &harness.committee, 0xC4); + + let (receiver, mut events, task) = harness.start_autonomous(1); + wait_ready(&mut events).await; + receiver.peer_connected(0).unwrap(); + receiver + .carrier(0, harness.envelope(&candidate, 0)) + .unwrap(); + + let mut authorized = false; + let deadline = + Instant::now() + SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1.saturating_mul(3); + while Instant::now() < deadline { + let remaining = deadline.saturating_duration_since(Instant::now()); + let event = match timeout(remaining, events.recv()).await { + Ok(Some(event)) => event, + Ok(None) => panic!("shadow actor stopped while checking empty application"), + Err(_) => break, + }; + match event { + ShadowServiceEventV1::AuthorizedApplicationObserved { + header: observed, + payload: None, + authorization_basis: + ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, + .. + } if observed == header => { + authorized = true; + receiver + .application_data_available(header.reference()) + .expect("empty authorized application must accept the DA callback"); } - if open_rounds[..online] - .iter() - .all(|round| *round >= target_open_round) - { - return; + ShadowServiceEventV1::Network { + message: NetworkMessage::RbcDagApplicationPayloadRequest(application), + .. + } if application == header.reference() => { + panic!("empty authorized application must not request payload bytes") } - if !progressed { - tokio::time::sleep(Duration::from_millis(1)).await; - } else { - tokio::task::yield_now().await; + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("empty authorized application was rejected: {error}") } + _ => {} } - }) - .await - .unwrap_or_else(|_| panic!("online prefix did not open round {target_open_round}")); + } + assert!(authorized, "empty application header was not authorized"); + stop(receiver, events, task).await; } - async fn stop( - handle: StarfishRbcDagShadowServiceHandleV1, - events: mpsc::Receiver, - task: JoinHandle<()>, - ) { - drop(events); - handle.shutdown().await.unwrap(); - task.await.unwrap(); - } + #[tokio::test] + async fn authorized_payload_recovery_is_peer_bound_verified_and_coalesced() { + let harness = Harness::new(); + let (header, payload) = application_header_and_payload(0, 1, 0xC4, &harness.committee); + let candidate = + round_one_application_candidate(0, header.clone(), &harness.committee, 0xC5); + let envelope = harness.envelope(&candidate, 0); - async fn startup_rejection(mut events: mpsc::Receiver) -> String { + let (receiver, mut events, task) = harness.start_autonomous(1); + wait_ready(&mut events).await; + receiver.peer_connected(0).unwrap(); + receiver.peer_connected(2).unwrap(); + receiver.carrier(0, envelope).unwrap(); + + let mut observed_header = false; + let mut requested = false; + while !observed_header || !requested { + match next_event(&mut events).await { + ShadowServiceEventV1::AuthorizedApplicationObserved { + header: observed, + payload: None, + authorization_basis: + ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, + .. + } if observed == header => observed_header = true, + ShadowServiceEventV1::Network { + recipient: 0, + message: NetworkMessage::RbcDagApplicationPayloadRequest(application), + } if application == header.reference() => requested = true, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("authorized header-only carrier was rejected: {error}") + } + _ => {} + } + } + + receiver + .application_payload_response( + 2, + RbcDagApplicationPayloadResponse { + application: header.reference(), + transaction_data: Arc::clone(&payload), + }, + ) + .unwrap(); + loop { + if let ShadowServiceEventV1::Rejected { + peer: Some(2), + error, + } = next_event(&mut events).await + { + assert!(error.contains("unrequested authority")); + break; + } + } + + receiver + .application_payload_response( + 0, + RbcDagApplicationPayloadResponse { + application: header.reference(), + transaction_data: Arc::clone(&payload), + }, + ) + .unwrap(); + loop { + if let ShadowServiceEventV1::AuthorizedApplicationObserved { + header: observed, + payload: Some(observed_payload), + .. + } = next_event(&mut events).await + { + assert_eq!(observed, header); + assert!(application_payloads_equal( + Some(observed_payload.as_ref()), + Some(payload.as_ref()) + )); + break; + } + } + + receiver + .application_payload_response( + 0, + RbcDagApplicationPayloadResponse { + application: header.reference(), + transaction_data: Arc::clone(&payload), + }, + ) + .unwrap(); + receiver + .verified_application_payload(header.reference(), Arc::clone(&payload)) + .unwrap(); + receiver + .application_payload_request(2, header.reference()) + .unwrap(); loop { match next_event(&mut events).await { - ShadowServiceEventV1::Rejected { peer: None, error } => return error, - ShadowServiceEventV1::Ready { .. } => { - panic!("invalid shadow startup became ready") + ShadowServiceEventV1::Network { + recipient: 2, + message: NetworkMessage::RbcDagApplicationPayloadResponse(response), + } => { + assert_eq!(response.application, header.reference()); + assert!(application_payloads_equal( + Some(response.transaction_data.as_ref()), + Some(payload.as_ref()) + )); + break; } + ShadowServiceEventV1::AuthorizedApplicationObserved { + payload: Some(_), .. + } => panic!("duplicate payload response emitted duplicate application work"), _ => {} } } + receiver + .application_payload_request(2, header.reference()) + .unwrap(); + loop { + if let ShadowServiceEventV1::Input { + kind: "application_payload_request", + outcome: "rate_limited", + } = next_event(&mut events).await + { + break; + } + } + stop(receiver, events, task).await; } - fn direct_header(author: AuthorityIndex, round: RoundNumber, marker: u8) -> RbcCanonicalHeader { - RbcCanonicalHeader::try_new( - author, - round, - (0..N) - .map(|authority| { - *VerifiedBlock::new_genesis(authority as AuthorityIndex).reference() - }) - .collect(), - Vec::new(), - u64::from(round) * 1_000 + u64::from(marker), - TransactionsCommitment::from_bytes([marker; 32]), + #[tokio::test] + async fn application_and_quarantine_maps_remain_bounded() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), ) - .unwrap() + .unwrap(); + let (state, _events, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + tokio::task::spawn_blocking(move || { + let mut state = state; + for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + let round = offset as RoundNumber + 1; + let header = direct_header(0, round, offset as u8); + state + .authorize_application( + BlockReference::new_test(0, 1_000 + round), + header, + None, + false, + None, + ShadowApplicationAuthorizationBasisV1::Delivered, + ) + .unwrap(); + } + assert_eq!( + state.authorized_applications.len(), + SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + ); + + for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + let round = offset as RoundNumber + 1; + state + .quarantine_application( + BlockReference::new_test(1, 2_000 + round), + BlockReference::new_test(1, round), + 1, + None, + ) + .unwrap(); + } + assert_eq!( + state.quarantined_application_payloads.len(), + SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + ); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn delayed_verified_payload_for_evicted_application_is_stale_not_rejected() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (state, mut events, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + tokio::task::spawn_blocking(move || { + let mut state = state; + let mut applications = Vec::new(); + for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + let round = offset as RoundNumber + 1; + let header = direct_header(0, round, offset as u8); + applications.push(header.reference()); + state + .authorize_application( + BlockReference::new_test(0, 3_000 + round), + header, + None, + false, + None, + ShadowApplicationAuthorizationBasisV1::Delivered, + ) + .unwrap(); + } + let evicted = applications + .into_iter() + .find(|application| !state.authorized_applications.contains_key(application)) + .expect("one authorized application must be evicted at capacity plus one"); + let delayed_payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![0xD3; 64]), + )])); + state + .desired_verified_application_payloads + .lock() + .insert(evicted, delayed_payload); + state.reconcile_verified_application_payloads(); + assert!( + state + .desired_verified_application_payloads + .lock() + .is_empty() + ); + assert!(!state.fatal); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + + let mut observed_stale = false; + while let Ok(event) = events.try_recv() { + match event { + ShadowServiceEventV1::Input { + kind: "verified_application_payload", + outcome: "stale_ignored", + } => observed_stale = true, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("delayed verified callback was rejected: {error}") + } + _ => {} + } + } + assert!(observed_stale, "stale callback outcome was not reported"); + } + + #[tokio::test] + async fn delayed_payload_response_for_evicted_application_is_stale_not_rejected() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (state, mut events, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + tokio::task::spawn_blocking(move || { + let mut state = state; + let evicted = BlockReference::new_test(1, 1); + let payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( + Transaction::new(vec![0xD4; 64]), + )])); + state + .handle_application_payload_response( + 1, + RbcDagApplicationPayloadResponse { + application: evicted, + transaction_data: payload, + }, + ) + .expect("late network completion must be an idempotent stale observation"); + assert!(!state.fatal); + state.core.shutdown().unwrap(); + }) + .await + .unwrap(); + + let mut observed_stale = false; + while let Ok(event) = events.try_recv() { + match event { + ShadowServiceEventV1::Input { + kind: "application_payload_response", + outcome: "stale_ignored", + } => observed_stale = true, + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("delayed payload response was rejected: {error}") + } + _ => {} + } + } + assert!( + observed_stale, + "stale network response outcome was not reported" + ); } fn round_one_candidate( @@ -2782,13 +6808,244 @@ mod tests { stop(handle, events, task).await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn instrumented_autonomous_service_records_assignment_and_queue_state() { + let harness = Harness::new(); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new(®istry, None, None, None); + metrics.metrics_active.store(true, Ordering::Relaxed); + let (handle, mut events, task) = + start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + Vec::new(), + Duration::from_secs(60 * 60), + ShadowWalSyncPolicyV1::EveryBatch, + Arc::clone(&metrics), + ) + .unwrap(); + wait_ready(&mut events).await; + + let application = direct_header(0, 1, 0xA1); + handle.local_header(&application).unwrap(); + let mut application_assigned = false; + timeout(EVENT_TIMEOUT, async { + loop { + match events.recv().await { + Some(ShadowServiceEventV1::ApplicationAssigned(reference)) => { + application_assigned = true; + assert_eq!(reference, application.reference()); + } + Some(ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrier(_), + }) => { + assert!( + application_assigned, + "the local producer gate must be released before network fan-out" + ); + break; + } + _ => {} + } + } + }) + .await + .expect("carrier transition did not publish its assignment and network events"); + assert!(application_assigned); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_latency_samples_total + .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT]) + .get(), + 1 + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_queue_depth + .with_label_values(&["local"]) + .get(), + 0 + ); + assert_eq!( + metrics + .starfish_rbc_dag_pipeline_queue_depth_max + .with_label_values(&["local"]) + .get(), + 1 + ); + assert_eq!( + metrics + .starfish_rbc_dag_projection_hol_state + .with_label_values(&["insufficient_lookahead"]) + .get(), + 1 + ); + + stop(handle, events, task).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn recovered_application_assignment_does_not_release_a_live_producer_gate() { + let harness = Harness::new(); + let application = direct_header(0, 1, 0xA2); + let (handle, mut events, task) = start_starfish_rbc_dag_autonomous_clock_service_v1( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + vec![application], + Duration::from_secs(60 * 60), + ShadowWalSyncPolicyV1::EveryBatch, + ) + .unwrap(); + + timeout(EVENT_TIMEOUT, async { + loop { + match events.recv().await { + Some(ShadowServiceEventV1::ApplicationAssigned(reference)) => panic!( + "recovered application {reference} has no live producer gate to release" + ), + Some(ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrier(_), + }) => break, + Some(ShadowServiceEventV1::Rejected { error, .. }) => { + panic!("recovered application was rejected: {error}") + } + _ => {} + } + } + }) + .await + .expect("recovered application was not assigned to the open carrier"); + + stop(handle, events, task).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn live_duplicate_upgrades_recovered_pending_application_to_exactly_one_ack() { + let harness = Harness::new(); + let application = direct_header(0, 1, 0xA3); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (mut state, mut events, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + state.pending_local.insert( + application.reference().round, + ShadowLocalCarrierV1::from_recovered_direct_header(&application), + ); + + let expected_reference = application.reference(); + let state = tokio::task::spawn_blocking(move || { + state.enqueue_local(ShadowLocalCarrierV1::from_direct_header(&application)); + assert!( + state + .pending_local + .get(&application.reference().round) + .is_some_and(|pending| pending.acknowledge_assignment) + ); + state.try_create_autonomous_carrier(); + state + }) + .await + .unwrap(); + let acknowledgments = std::iter::from_fn(|| events.try_recv().ok()) + .filter_map(|event| match event { + ShadowServiceEventV1::ApplicationAssigned(reference) => Some(reference), + _ => None, + }) + .collect::>(); + assert_eq!(acknowledgments, vec![expected_reference]); + state.core.shutdown().unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn paused_autonomous_clock_uses_one_idempotent_fresh_activation_epoch() { + let harness = Harness::new(); + let heartbeat_interval = Duration::from_millis(120); + let (handle, mut events, task) = + harness.start_autonomous_paused_with_interval(0, heartbeat_interval); + wait_ready(&mut events).await; + + // Neither the dormant timer nor an explicitly injected stale tick may + // fix a carrier while the external coordinated-start barrier is shut. + tokio::time::sleep(heartbeat_interval.saturating_mul(4)).await; + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + let inspection = handle.inspect_carrier_sync().await.unwrap(); + assert_eq!(inspection.open_round, 1); + while let Ok(event) = events.try_recv() { + assert!(!matches!(event, ShadowServiceEventV1::ClockActivated)); + assert!( + !matches!(event, ShadowServiceEventV1::Network { .. }), + "paused autonomous clock emitted protocol network traffic: {event:?}" + ); + } + + let activated_at = Instant::now(); + handle.activate_clock().await.unwrap(); + handle.activate_clock().await.unwrap(); + + let mut activation_events = 0; + let early_network = timeout(heartbeat_interval / 2, async { + loop { + match events.recv().await { + Some(ShadowServiceEventV1::ClockActivated) => activation_events += 1, + Some(ShadowServiceEventV1::Network { .. }) => return, + Some(ShadowServiceEventV1::Rejected { error, .. }) => { + panic!("activated clock rejected input: {error}") + } + Some(_) => {} + None => panic!("activated clock stopped before its first heartbeat"), + } + } + }) + .await; + assert!( + early_network.is_err(), + "activation inherited a stale heartbeat deadline" + ); + assert_eq!(activation_events, 1, "activation was not idempotent"); + + timeout( + heartbeat_interval.saturating_mul(3), + next_carrier(&mut events, 1), + ) + .await + .expect("fresh activation epoch did not produce its first heartbeat"); + assert!( + activated_at.elapsed() >= heartbeat_interval.saturating_sub(Duration::from_millis(20)), + "first heartbeat did not wait one fresh interval" + ); + + stop(handle, events, task).await; + } + async fn assert_autonomous_zero_load_progress(n: usize) { let harness = Harness::new_with_n(n); let mut handles = Vec::new(); let mut events = Vec::new(); let mut tasks = Vec::new(); for authority in 0..n as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous(authority); + // Exercise the production 600 ms clock so the shared 30 ms + // normal-carrier permit advances under real wall time. The test + // still injects heartbeat triggers explicitly for determinism. + let (handle, mut node_events, task) = + harness.start_autonomous_with_interval(authority, Duration::from_millis(600)); loop { match next_event(&mut node_events).await { ShadowServiceEventV1::Ready { autonomous_clock } => { @@ -2813,6 +7070,8 @@ mod tests { let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; + let mut max_buffered_authenticated = 0; + let mut rejections = Vec::new(); for (authority, handle) in handles.iter().enumerate() { for peer in 0..n { if peer != authority { @@ -2834,9 +7093,26 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, fixed_round + 1, + EVENT_TIMEOUT, ) .await; + if fixed_round == 2 { + for (authority, handle) in handles.iter().enumerate() { + let (optimistic_promises, certified_deliveries) = + handle.inspect_rbc_progress().await.unwrap(); + assert!( + optimistic_promises > 0, + "authority {authority} did not lock an optimistic ECHO promise by the round-two carrier" + ); + assert_eq!( + certified_deliveries, 0, + "authority {authority} certified a carrier before the later READY round" + ); + } + } } assert!( @@ -2856,8 +7132,9 @@ mod tests { "clean projection did not decide: vertices={projected_vertices}, rounds={open_rounds:?}" ); assert_eq!( - sync_requests, 0, - "healthy proactive rounds must not trigger repair polling" + sync_requests, + n * (n - 1), + "each connection must repair exactly the round-one carrier fixed before topology registration" ); drop(events); @@ -2902,7 +7179,8 @@ mod tests { let mut events = Vec::new(); let mut tasks = Vec::new(); for authority in 0..N as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous(authority); + let (handle, mut node_events, task) = + harness.start_autonomous_with_interval(authority, Duration::from_millis(600)); wait_ready(&mut node_events).await; handles.push(handle); events.push(node_events); @@ -2937,6 +7215,8 @@ mod tests { let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; + let mut max_buffered_authenticated = 0; + let mut rejections = Vec::new(); pump_autonomous_until_round( &handles, &mut events, @@ -2947,13 +7227,16 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, 5, + EVENT_TIMEOUT, ) .await; // The first directly committed consensus frontier may predate the // round-one application deliveries. Advance enough certified carrier // rounds for a later committed frontier to include that closed prefix. - for fixed_round in 5..=18 { + for fixed_round in 5..=40 { for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -2967,9 +7250,21 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, fixed_round + 1, + EVENT_TIMEOUT, ) .await; + if committed_frontiers.iter().all(|commits| { + commits + .iter() + .flat_map(|delta| delta.applications.iter().map(RbcCanonicalHeader::reference)) + .collect::>() + == expected + }) { + break; + } } assert!( @@ -2982,7 +7277,11 @@ mod tests { open_rounds.iter().all(|round| *round >= 5), "the carrier DAG must keep advancing after application delivery" ); - assert_eq!(sync_requests, 0); + assert_eq!( + sync_requests, + N * (N - 1), + "each connection must repair exactly the round-one carrier fixed before topology registration" + ); assert!( projected_vertices >= N, "empty embedded applications must not stall clean projection" @@ -3134,13 +7433,17 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn autonomous_exact_sync_closes_a_multi_round_gap() { + async fn pipelined_exact_sync_closes_gap_beyond_retention_while_producers_run() { let harness = Harness::new(); let mut handles = Vec::new(); let mut events = Vec::new(); let mut tasks = Vec::new(); for authority in 0..N as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous(authority); + let (handle, mut node_events, task) = harness.start_autonomous_with_policy( + authority, + Duration::from_millis(600), + ShadowWalSyncPolicyV1::OnShutdown, + ); wait_ready(&mut node_events).await; handles.push(handle); events.push(node_events); @@ -3150,8 +7453,9 @@ mod tests { // Establish round one for all validators, then let a quorum advance // while authority 3 is offline and receives none of the proactive // carriers. Starting the gap at round two makes reconnect request - // exact repair immediately; the one-hour normal heartbeat still - // cannot help with the later repaired rounds. + // exact repair immediately. The normal pacer drives the healthy + // prefix, while the later lagger convergence must use the bounded + // validity-backed repair lane rather than wait for normal permits. for (authority, handle) in handles.iter().enumerate() { for peer in 0..N { if peer != authority { @@ -3166,6 +7470,8 @@ mod tests { let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; + let mut max_buffered_authenticated = 0; + let mut rejections = Vec::new(); for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -3179,7 +7485,10 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, 2, + EVENT_TIMEOUT, ) .await; for authority in 0..3 { @@ -3188,7 +7497,7 @@ mod tests { .peer_disconnected(authority as AuthorityIndex) .unwrap(); } - for fixed_round in 2..=8 { + for fixed_round in 2..=72 { for handle in &handles[..3] { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -3202,12 +7511,12 @@ mod tests { .await; } assert_eq!(open_rounds[3], 2); - assert!(open_rounds[..3].iter().all(|round| *round >= 9)); + assert!(open_rounds[..3].iter().all(|round| *round >= 73)); // Reconnect, fix the lagging node's first local slot, and let the // healthy quorum open one more round. Exact responses then drive an - // immediate local heartbeat per repaired round; the one-hour normal - // timer cannot be responsible for convergence. + // immediate local heartbeat per repaired round; convergence must + // outrun the independently paced healthy producers. for (authority, handle) in handles.iter().enumerate() { for peer in 0..N { if peer != authority { @@ -3218,9 +7527,26 @@ mod tests { handles[3] .send(ShadowServiceMessageV1::HeartbeatTick) .unwrap(); - for handle in &handles[..3] { - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - } + let catch_up_initial_gap = open_rounds[..3] + .iter() + .copied() + .min() + .unwrap() + .saturating_sub(open_rounds[3]); + let producer_running = Arc::new(AtomicBool::new(true)); + let producer_flag = Arc::clone(&producer_running); + let producer_handles = handles[..3].to_vec(); + let producer = tokio::spawn(async move { + while producer_flag.load(Ordering::Relaxed) { + for handle in &producer_handles { + handle + .send_reliably(ShadowServiceMessageV1::HeartbeatTick) + .await + .unwrap(); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }); let sync_requests_before_catch_up = sync_requests; pump_autonomous_until_round( @@ -3233,15 +7559,213 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, - 10, + &mut max_buffered_authenticated, + &mut rejections, + 80, + Duration::from_secs(30), ) .await; + let inspect_handle = handles[3].clone(); + let inspection_task = tokio::spawn(async move { + inspect_handle + .inspect_carrier_sync() + .await + .expect("lagging actor stopped before live catch-up inspection") + }); + timeout(Duration::from_secs(5), async { + while !inspection_task.is_finished() { + let observed_minimum = open_rounds.iter().copied().min().unwrap_or_default(); + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut application_deliveries, + &mut committed_frontiers, + &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, + observed_minimum, + Duration::from_secs(1), + ) + .await; + tokio::task::yield_now().await; + } + }) + .await + .expect("live catch-up inspection was starved behind actor events"); + let live_inspection = inspection_task.await.unwrap(); + let live_lagger_round = open_rounds[3].max(live_inspection.open_round); + let live_healthy_minimum = open_rounds[..3].iter().copied().min().unwrap(); + // A fast run may already have reached and drained the validity-backed + // target, in which case the actor correctly clears the episode. Treat + // that stronger state as a zero target gap at the inspected round. + let live_target = live_inspection.target.unwrap_or(live_lagger_round); + let live_healthy_gap = live_healthy_minimum.saturating_sub(live_lagger_round); + let live_target_gap = live_target.saturating_sub(live_lagger_round); + assert!( + live_lagger_round.saturating_sub(2) > EXECUTABLE_MODEL_BUFFER_WINDOW_V1, + "lagger did not repair beyond the normal retention horizon: live={open_rounds:?}, inspection={live_inspection:?}" + ); + assert!( + live_healthy_gap < catch_up_initial_gap, + "lagger gap did not shrink while producers remained live: initial_gap={catch_up_initial_gap}, live={open_rounds:?}, inspection={live_inspection:?}" + ); + assert!( + live_target_gap <= live_healthy_gap, + "validity-backed target escaped beyond the healthy live high-water: live={open_rounds:?}, inspection={live_inspection:?}" + ); + + // A single shrinking observation is not sufficient: repair could + // still plateau permanently outside the bounded exact-slot window. + // Keep the producers live and route fairly until the lagger is within + // one pipeline of both the validity-backed target and the observed + // healthy minimum. A fully drained episode that already reached and + // cleared its target is the stronger equivalent outcome. + let pipeline_depth = carrier_sync_pipeline_depth(N) as RoundNumber; + let (bounded_inspection, bounded_lagger_round) = + timeout(Duration::from_secs(5), async { + loop { + let inspect_handle = handles[3].clone(); + let inspection_task = tokio::spawn(async move { + inspect_handle.inspect_carrier_sync().await.expect( + "lagging actor stopped before bounded catch-up inspection", + ) + }); + while !inspection_task.is_finished() { + let observed_minimum = + open_rounds.iter().copied().min().unwrap_or_default(); + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut application_deliveries, + &mut committed_frontiers, + &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, + observed_minimum, + Duration::from_secs(1), + ) + .await; + tokio::task::yield_now().await; + } + let inspection = inspection_task.await.unwrap(); + let lagger_round = open_rounds[3].max(inspection.open_round); + let healthy_minimum = open_rounds[..3].iter().copied().min().unwrap(); + let healthy_gap = healthy_minimum.saturating_sub(lagger_round); + let target_gap = inspection + .target + .map(|target| target.saturating_sub(lagger_round)) + .unwrap_or_default(); + let episode_cleared = inspection.target.is_none() + && inspection.outstanding == 0 + && inspection.desired_responses == 0; + if episode_cleared + || (healthy_gap <= pipeline_depth && target_gap <= pipeline_depth) + { + break (inspection, lagger_round); + } + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "live exact repair plateaued outside pipeline depth {pipeline_depth}: open={open_rounds:?}, first_inspection={live_inspection:?}" + ) + }); + producer_running.store(false, Ordering::Relaxed); + producer.await.unwrap(); assert!( sync_requests > sync_requests_before_catch_up, "catch-up must use exact-slot repair" ); - assert!(open_rounds.iter().all(|round| *round >= 10)); - + // Maintenance timers may keep publishing benign state observations, + // so quiescence cannot mean an indefinitely empty event channel. + // Fairly route one event per node for one complete exact-slot budget; + // every staged network event is delivered before each pass returns. + for _ in 0..SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 { + let observed_minimum = open_rounds.iter().copied().min().unwrap_or_default(); + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut application_deliveries, + &mut committed_frontiers, + &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, + observed_minimum, + Duration::from_secs(1), + ) + .await; + } + let inspect_handle = handles[3].clone(); + let final_inspection_task = tokio::spawn(async move { + inspect_handle + .inspect_carrier_sync() + .await + .expect("lagging actor stopped before final catch-up inspection") + }); + timeout(Duration::from_secs(2), async { + while !final_inspection_task.is_finished() { + let observed_minimum = open_rounds.iter().copied().min().unwrap_or_default(); + pump_autonomous_until_round( + &handles, + &mut events, + &mut open_rounds, + &mut deliveries, + &mut application_deliveries, + &mut committed_frontiers, + &mut sync_requests, + &mut projected_vertices, + &mut projected_decisions, + &mut max_buffered_authenticated, + &mut rejections, + observed_minimum, + Duration::from_secs(1), + ) + .await; + tokio::task::yield_now().await; + } + }) + .await + .expect("final catch-up inspection timed out behind actor events"); + let final_inspection = final_inspection_task.await.unwrap(); + assert!( + final_inspection.open_round >= bounded_lagger_round, + "short router drain regressed catch-up progress: bounded={bounded_inspection:?}, final={final_inspection:?}" + ); + assert!(max_buffered_authenticated > 0); + let buffered_authenticated_capacity = usize::try_from( + EXECUTABLE_MODEL_BUFFER_WINDOW_V1.saturating_sub(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1), + ) + .unwrap_or(usize::MAX) + .saturating_mul(N.saturating_sub(1)); + assert!( + max_buffered_authenticated <= buffered_authenticated_capacity, + "pipelined exact repair exceeded the executable model's authenticated retention capacity: observed={max_buffered_authenticated}, capacity={buffered_authenticated_capacity}" + ); + assert!( + final_inspection.max_outstanding <= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, + "outstanding exact slots exceeded the global cap: {final_inspection:?}" + ); + assert!( + final_inspection.max_desired_responses <= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, + "coalesced exact responses exceeded the global cap: {final_inspection:?}" + ); + assert!( + rejections.is_empty(), + "valid catch-up produced rejections: {rejections:?}" + ); drop(events); for handle in &handles { handle.shutdown().await.unwrap(); @@ -3596,6 +8120,79 @@ mod tests { stop(receiver, receiver_events, receiver_task).await; } + #[tokio::test] + async fn poisoned_application_payload_waits_for_exact_delivery() { + let harness = Harness::new(); + let (header, payload) = application_header_and_payload(3, 1, 0xC2, &harness.committee); + let target = round_one_application_candidate(3, header.clone(), &harness.committee, 0xC3); + let mut envelope = harness.envelope(&target, 3); + envelope.application_payload = Some(Arc::clone(&payload)); + envelope.authentication_sidecar[3 + 2 * MAC_TAG_SIZE] ^= 1; + + let (receiver, mut events, task) = harness.start(2, Vec::new()); + wait_ready(&mut events).await; + receiver.carrier(3, envelope).unwrap(); + let mut retained = false; + let mut rejected = false; + while !retained || !rejected { + match next_event(&mut events).await { + ShadowServiceEventV1::Input { + kind: "carrier", + outcome: "retained_unauthenticated", + } => retained = true, + ShadowServiceEventV1::Rejected { peer: Some(3), .. } => rejected = true, + ShadowServiceEventV1::AuthorizedApplicationObserved { .. } => { + panic!("poisoned receiver MAC must not authorize its payload") + } + _ => {} + } + } + + for author in [0, 1] { + let phase = phase_carrier( + author, + RbcPhaseStatementV1::Ready { + target: target.reference(), + }, + &harness.committee, + ); + receiver + .carrier(author, harness.envelope(&phase, author)) + .unwrap(); + } + let mut delivered = false; + loop { + match next_event(&mut events).await { + ShadowServiceEventV1::Delivered(identity) + if identity.author == 3 && identity.round == 1 => + { + delivered = true; + } + ShadowServiceEventV1::AuthorizedApplicationObserved { + carrier, + header: observed, + payload: Some(observed_payload), + authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, + } => { + assert!(delivered, "authorization must follow the Delivered effect"); + assert_eq!(carrier, target.reference()); + assert_eq!(observed, header); + assert!(application_payloads_equal( + Some(observed_payload.as_ref()), + Some(payload.as_ref()) + )); + break; + } + ShadowServiceEventV1::Rejected { + peer: Some(peer), + error, + } if peer != 3 => panic!("delivery evidence was rejected: {error}"), + _ => {} + } + } + stop(receiver, events, task).await; + } + #[tokio::test] async fn reconnect_replays_exact_persisted_envelope() { let harness = Harness::new(); @@ -3812,6 +8409,67 @@ mod tests { stop(handle, events, task).await; } + #[test] + fn verified_payload_callback_coalesces_when_notification_queue_is_full() { + let harness = Harness::new(); + let (header, payload) = application_header_and_payload(0, 1, 0xD1, &harness.committee); + let (other_header, other_payload) = + application_header_and_payload(0, 2, 0xD2, &harness.committee); + let (sender, mut receiver) = mpsc::channel(1); + let desired_verified_application_payloads = Arc::new(Mutex::new(BTreeMap::new())); + let invalidated_by_overload = Arc::new(Mutex::new(None)); + let handle = StarfishRbcDagShadowServiceHandleV1 { + sender, + mode: ShadowServiceModeV1::DirectMirror, + max_sidecar_size: 3 + N * MAC_TAG_SIZE, + own_authority: 0, + committee_size: N, + input_capacity: 1, + desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), + desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), + desired_verified_application_payloads: Arc::clone( + &desired_verified_application_payloads, + ), + desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), + desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), + invalidated_by_overload: Arc::clone(&invalidated_by_overload), + }; + + handle.send(ShadowServiceMessageV1::RetryRecovery).unwrap(); + handle + .verified_application_payload(header.reference(), Arc::clone(&payload)) + .unwrap(); + handle + .verified_application_payload(header.reference(), Arc::clone(&payload)) + .unwrap(); + + assert_eq!(desired_verified_application_payloads.lock().len(), 1); + assert_eq!(*invalidated_by_overload.lock(), None); + assert!(matches!( + receiver.try_recv(), + Ok(ShadowServiceMessageV1::RetryRecovery) + )); + assert!(matches!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + + handle + .verified_application_payload(other_header.reference(), Arc::clone(&other_payload)) + .unwrap(); + assert!(matches!( + receiver.try_recv(), + Ok(ShadowServiceMessageV1::VerifiedApplicationPayloadsChanged) + )); + assert_eq!(desired_verified_application_payloads.lock().len(), 2); + assert!(matches!( + handle.verified_application_payload(header.reference(), other_payload), + Err(ShadowServiceErrorV1::ConflictingApplicationPayload(application)) + if application == header.reference() + )); + } + #[tokio::test] async fn bounded_input_reports_overload_but_shutdown_waits_for_capacity() { let input_capacity = shadow_input_capacity(N, ShadowServiceModeV1::DirectMirror).unwrap(); @@ -3824,6 +8482,9 @@ mod tests { committee_size: N, input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), + desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), + desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), @@ -3835,6 +8496,7 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![round as u8], authentication_sidecar: Vec::new(), + application_payload: None, }, ) .unwrap(); @@ -3845,6 +8507,7 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![0xFF], authentication_sidecar: Vec::new(), + application_payload: None, }, ), Err(ShadowServiceErrorV1::Overloaded { @@ -3875,6 +8538,9 @@ mod tests { committee_size: N, input_capacity: 1, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), + desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), + desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), @@ -3885,6 +8551,7 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![0; MAX_CARRIER_CONTENT_SIZE_V1 + 1], authentication_sidecar: Vec::new(), + application_payload: None, }, ), Err(ShadowServiceErrorV1::InputTooLarge { @@ -3910,6 +8577,9 @@ mod tests { committee_size: LARGE_N, input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), + desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), + desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), + desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::clone(&invalidated), @@ -3921,6 +8591,7 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![peer as u8], authentication_sidecar: Vec::new(), + application_payload: None, }, ) .unwrap(); diff --git a/crates/starfish-core/src/stat.rs b/crates/starfish-core/src/stat.rs index 34f6b540..c18b81fe 100644 --- a/crates/starfish-core/src/stat.rs +++ b/crates/starfish-core/src/stat.rs @@ -94,6 +94,16 @@ impl PreciseHistogram { self.points.clear(); } + /// Reset both the current distribution and its lifetime aggregates. + /// Used only when a benchmark establishes a new explicit observation + /// epoch; periodic reporting continues to use `clear` and preserves its + /// historical running totals. + pub fn reset(&mut self) { + self.points.clear(); + self.sum = T::default(); + self.count = 0; + } + fn pct1000_index(&self, pct1000: usize) -> usize { debug_assert!(pct1000 < 1000); self.points.len() * pct1000 / 1000 diff --git a/crates/starfish-core/src/store.rs b/crates/starfish-core/src/store.rs index 31ad4fbd..32607a20 100644 --- a/crates/starfish-core/src/store.rs +++ b/crates/starfish-core/src/store.rs @@ -3,12 +3,208 @@ use std::io; +use serde::{Deserialize, Serialize}; + use crate::{ + crypto::BLOCK_DIGEST_SIZE, dag_state::CommitData, data::Data, - types::{BlockReference, ProvableShard, RoundNumber, VerifiedBlock}, + types::{BlockReference, MAX_COMMITTEE_SIZE, ProvableShard, RoundNumber, VerifiedBlock}, }; +/// Durable acknowledgement that Core applied one RBC-DAG committed frontier. +/// +/// This is a single, bounded latest-value cursor rather than an append-only +/// history. `carrier_anchor` is intentionally independent of application block +/// storage: an RBC-DAG consensus carrier is not necessarily a Core block. The +/// per-authority watermark lets Core restore commit progress without scanning +/// application references. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcDagFrontierReceipt { + pub(crate) carrier_anchor: BlockReference, + /// Monotone one-based position in the authoritative frontier output + /// stream. This is intentionally independent of the anchor's logical + /// consensus round, which may regress when a later certifier resolves an + /// older leader. + pub(crate) output_sequence: RoundNumber, + pub(crate) committed_rounds: Vec, +} + +impl RbcDagFrontierReceipt { + const ENCODING_MAGIC: [u8; 4] = *b"RDF1"; + const FIXED_ENCODED_LEN: usize = Self::ENCODING_MAGIC.len() + + std::mem::size_of::() + + std::mem::size_of::() + + BLOCK_DIGEST_SIZE + + std::mem::size_of::() + + std::mem::size_of::(); + const MAX_ENCODED_LEN: usize = + Self::FIXED_ENCODED_LEN + MAX_COMMITTEE_SIZE as usize * std::mem::size_of::(); + + fn validate(&self, error_kind: io::ErrorKind) -> io::Result<()> { + if self.output_sequence == 0 { + return Err(io::Error::new( + error_kind, + "RBC-DAG frontier receipt output sequence must be nonzero", + )); + } + if self.committed_rounds.is_empty() + || self.committed_rounds.len() > usize::from(MAX_COMMITTEE_SIZE) + { + return Err(io::Error::new( + error_kind, + format!( + "RBC-DAG frontier receipt must contain 1..={MAX_COMMITTEE_SIZE} committed-round watermarks, got {}", + self.committed_rounds.len() + ), + )); + } + if self.carrier_anchor.authority >= MAX_COMMITTEE_SIZE { + return Err(io::Error::new( + error_kind, + format!( + "RBC-DAG frontier receipt carrier authority {} exceeds the maximum {}", + self.carrier_anchor.authority, + MAX_COMMITTEE_SIZE - 1 + ), + )); + } + Ok(()) + } + + pub(crate) fn to_bytes(&self) -> io::Result> { + self.validate(io::ErrorKind::InvalidInput)?; + + let mut bytes = Vec::with_capacity( + Self::FIXED_ENCODED_LEN + + self.committed_rounds.len() * std::mem::size_of::(), + ); + bytes.extend_from_slice(&Self::ENCODING_MAGIC); + bytes.extend_from_slice(&self.carrier_anchor.round.to_le_bytes()); + bytes.extend_from_slice(&self.carrier_anchor.authority.to_le_bytes()); + bytes.extend_from_slice(self.carrier_anchor.digest.as_array()); + bytes.extend_from_slice(&self.output_sequence.to_le_bytes()); + let watermark_count = u16::try_from(self.committed_rounds.len()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "RBC-DAG frontier receipt watermark count is not encodable", + ) + })?; + bytes.extend_from_slice(&watermark_count.to_le_bytes()); + for round in &self.committed_rounds { + bytes.extend_from_slice(&round.to_le_bytes()); + } + + Ok(bytes) + } + + pub(crate) fn from_bytes(bytes: &[u8]) -> io::Result { + if !(Self::FIXED_ENCODED_LEN..=Self::MAX_ENCODED_LEN).contains(&bytes.len()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "stored RBC-DAG frontier receipt has an invalid encoded length", + )); + } + if bytes[..Self::ENCODING_MAGIC.len()] != Self::ENCODING_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "stored RBC-DAG frontier receipt has an unsupported encoding", + )); + } + + let mut cursor = Self::ENCODING_MAGIC.len(); + let take_u32 = |bytes: &[u8], cursor: &mut usize| { + let mut encoded = [0; std::mem::size_of::()]; + let end = *cursor + std::mem::size_of::(); + encoded.copy_from_slice(&bytes[*cursor..end]); + *cursor = end; + u32::from_le_bytes(encoded) + }; + let take_u16 = |bytes: &[u8], cursor: &mut usize| { + let mut encoded = [0; std::mem::size_of::()]; + let end = *cursor + std::mem::size_of::(); + encoded.copy_from_slice(&bytes[*cursor..end]); + *cursor = end; + u16::from_le_bytes(encoded) + }; + + let carrier_round = take_u32(bytes, &mut cursor); + let carrier_authority = take_u16(bytes, &mut cursor); + let mut carrier_digest = [0; BLOCK_DIGEST_SIZE]; + carrier_digest.copy_from_slice(&bytes[cursor..cursor + BLOCK_DIGEST_SIZE]); + cursor += BLOCK_DIGEST_SIZE; + let output_sequence = take_u32(bytes, &mut cursor); + let watermark_count = usize::from(take_u16(bytes, &mut cursor)); + if watermark_count == 0 || watermark_count > usize::from(MAX_COMMITTEE_SIZE) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "stored RBC-DAG frontier receipt has an invalid watermark count", + )); + } + let expected_len = + Self::FIXED_ENCODED_LEN + watermark_count * std::mem::size_of::(); + if bytes.len() != expected_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "stored RBC-DAG frontier receipt length does not match its watermark count", + )); + } + + let mut committed_rounds = Vec::with_capacity(watermark_count); + for _ in 0..watermark_count { + committed_rounds.push(take_u32(bytes, &mut cursor)); + } + let receipt = Self { + carrier_anchor: BlockReference { + round: carrier_round, + authority: carrier_authority, + digest: carrier_digest.into(), + }, + output_sequence, + committed_rounds, + }; + receipt.validate(io::ErrorKind::InvalidData)?; + Ok(receipt) + } +} + +/// Validate the exact atomic frontier write shape shared by every backend. +/// A nonempty batch contains precisely the current delta under its carrier +/// anchor; an empty batch is an explicit control-only marker. +pub(crate) fn validate_rbc_dag_frontier_commit_batch( + committed_sub_dags: &[CommitData], + receipt: &RbcDagFrontierReceipt, +) -> io::Result<()> { + receipt.validate(io::ErrorKind::InvalidInput)?; + if committed_sub_dags.len() > 1 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RBC-DAG frontier receipt batch must contain at most one application commit", + )); + } + if let Some(commit) = committed_sub_dags.first() { + if commit.sub_dag.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RBC-DAG frontier application commit must contain at least one application", + )); + } + if commit.leader != receipt.carrier_anchor { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RBC-DAG frontier application commit leader must equal the receipt carrier anchor", + )); + } + if commit.committed_rounds != receipt.committed_rounds { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RBC-DAG frontier application commit watermarks must equal the receipt watermarks", + )); + } + } + Ok(()) +} + /// Backend-agnostic storage interface for consensus blocks and commit data. /// /// Implementations must be thread-safe (`Send + Sync`). @@ -34,8 +230,22 @@ pub trait Store: Send + Sync + 'static { fn store_commits(&self, committed_sub_dags: Vec) -> io::Result<()>; + /// Atomically persist zero or more application commits and advance the + /// latest applied RBC-DAG frontier cursor. The carrier anchor is a + /// consensus-layer identity and is intentionally independent of any + /// application commit leader. A successful receipt read implies that all + /// commit data passed to this call is durable in the same batch. + fn store_commits_with_rbc_dag_receipt( + &self, + committed_sub_dags: Vec, + receipt: RbcDagFrontierReceipt, + ) -> io::Result<()>; + fn get_commit(&self, reference: &BlockReference) -> io::Result>; + /// Point-read the latest applied RBC-DAG frontier cursor in O(1). + fn read_latest_rbc_dag_frontier_receipt(&self) -> io::Result>; + // -- Component-level writes (pre-serialized) -- // Accept raw bincode bytes produced off the core thread by // `VerifiedBlock::preserialize()` or shard reconstructor workers. @@ -78,3 +288,81 @@ pub trait Store: Send + Sync + 'static { from_round: RoundNumber, ) -> io::Result>; } + +#[cfg(test)] +mod tests { + use super::{RbcDagFrontierReceipt, validate_rbc_dag_frontier_commit_batch}; + use crate::{dag_state::CommitData, types::BlockReference}; + + #[test] + fn rbc_dag_receipt_encoding_is_exact_and_bounded() { + let receipt = RbcDagFrontierReceipt { + carrier_anchor: BlockReference::new_test(3, 255), + output_sequence: 256, + committed_rounds: vec![250, 251, 252, 253], + }; + let encoded = receipt.to_bytes().unwrap(); + assert_eq!( + encoded.len(), + RbcDagFrontierReceipt::FIXED_ENCODED_LEN + + receipt.committed_rounds.len() * std::mem::size_of::() + ); + assert_eq!( + RbcDagFrontierReceipt::from_bytes(&encoded).unwrap(), + receipt + ); + + let mut trailing_byte = encoded.clone(); + trailing_byte.push(0); + assert_eq!( + RbcDagFrontierReceipt::from_bytes(&trailing_byte) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidData + ); + + let mut mismatched_count = encoded; + let count_offset = RbcDagFrontierReceipt::FIXED_ENCODED_LEN - std::mem::size_of::(); + mismatched_count[count_offset..count_offset + std::mem::size_of::()] + .copy_from_slice(&5u16.to_le_bytes()); + assert_eq!( + RbcDagFrontierReceipt::from_bytes(&mismatched_count) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidData + ); + + let zero_sequence = RbcDagFrontierReceipt { + carrier_anchor: BlockReference::new_test(0, 1), + output_sequence: 0, + committed_rounds: vec![0; 4], + }; + assert_eq!( + zero_sequence.to_bytes().unwrap_err().kind(), + std::io::ErrorKind::InvalidInput + ); + } + + #[test] + fn rbc_dag_frontier_commit_rejects_present_empty_application_data() { + let anchor = BlockReference::new_test(2, 7); + let receipt = RbcDagFrontierReceipt { + carrier_anchor: anchor, + output_sequence: 1, + committed_rounds: vec![0; 4], + }; + let empty_commit = CommitData { + leader: anchor, + sub_dag: Vec::new(), + committed_rounds: receipt.committed_rounds.clone(), + }; + + assert_eq!( + validate_rbc_dag_frontier_commit_batch(&[empty_commit], &receipt) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidInput + ); + assert!(validate_rbc_dag_frontier_commit_batch(&[], &receipt).is_ok()); + } +} diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index cf454cb6..9b7d4a84 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -13,7 +13,7 @@ use crate::{ bls_certificate_aggregator::{CertificateEvent, apply_certificate_events}, bls_service::BlsServiceMessage, consensus::{CommitMetastate, linearizer::CommittedSubDag}, - core::Core, + core::{Core, RbcDagFrontierApplyError, RbcDagFrontierApplyOutcome}, dag_state::{DagState, DataSource}, data::Data, metrics::Metrics, @@ -71,6 +71,15 @@ pub struct Syncer { starfish_rbc_service: Option, starfish_rbc_dag_shadow_service: Option, rbc_dag_frontier_authority: bool, + /// Production remains closed until the ordered service event bridge has + /// drained startup recovery and persisted every replayed frontier before + /// processing `Ready`. + rbc_dag_authority_ready: bool, + /// Exact locally produced application header waiting for durable carrier + /// assignment. Authoritative RBC-DAG mode permits only one outstanding + /// application so a fast legacy proposal clock cannot outrun the carrier + /// actor or turn a fixed queue size into a protocol parameter. + rbc_dag_pending_application: Option, } pub trait SyncerSignals: Send + Sync { @@ -85,12 +94,9 @@ pub trait CommitObserver: Send + Sync { committed_leaders: Vec<(Data, Option)>, ) -> Vec; - fn handle_rbc_dag_commit( - &mut self, - dag_state: &DagState, - anchor: BlockReference, - applications: &[BlockReference], - ) -> Vec; + /// Observe an RBC-DAG frontier only after Core atomically persisted its + /// application commit (if any) and durable frontier receipt. + fn handle_rbc_dag_commit(&mut self, committed: &[CommittedSubDag]); fn recover_committed( &mut self, @@ -103,7 +109,7 @@ pub trait CommitObserver: Send + Sync { impl Syncer { pub fn new( - mut core: Core, + core: Core, signals: S, commit_observer: C, metrics: Arc, @@ -113,9 +119,6 @@ impl Syncer { starfish_rbc_dag_shadow_service: Option, rbc_dag_frontier_authority: bool, ) -> Self { - if rbc_dag_frontier_authority { - core.enable_rbc_dag_application_production(); - } let committee_size = core.committee().len(); let own_stake = core .committee() @@ -137,6 +140,8 @@ impl Syncer { starfish_rbc_service, starfish_rbc_dag_shadow_service, rbc_dag_frontier_authority, + rbc_dag_authority_ready: !rbc_dag_frontier_authority, + rbc_dag_pending_application: None, } } @@ -148,8 +153,32 @@ impl Syncer { Vec, AHashSet, Vec, + ) { + if self.rbc_dag_frontier_authority { + tracing::warn!( + %source, + count = blocks.len(), + "Rejected generic block ingress while embedded RBC-DAG authority is active" + ); + return (Vec::new(), AHashSet::new(), Vec::new()); + } + self.add_blocks_inner(blocks, source) + } + + fn add_blocks_inner( + &mut self, + blocks: Vec<(Data, Option)>, + source: DataSource, + ) -> ( + Vec, + AHashSet, + Vec, ) { let previous_rounds = self.capture_rounds(); + let mut materialization_candidates = blocks + .iter() + .map(|(block, _)| *block.reference()) + .collect::>(); // todo: when block is updated we might return false here and it can make // committing longer let ( @@ -159,6 +188,8 @@ impl Syncer { used_additional_blocks, processed_blocks, ) = self.core.add_blocks(blocks, source); + materialization_candidates.extend(processed_blocks.iter().map(|block| *block.reference())); + self.notify_materialized_shadow_applications(materialization_candidates); if !processed_blocks.is_empty() { let block_refs: Vec<_> = processed_blocks.iter().map(|b| *b.reference()).collect(); self.send_sailfish_message(SailfishServiceMessage::ProcessBlocks(block_refs)); @@ -188,10 +219,32 @@ impl Syncer { &mut self, headers: Vec>, source: DataSource, + ) -> (AHashSet, Vec) { + if self.rbc_dag_frontier_authority { + tracing::warn!( + %source, + count = headers.len(), + "Rejected generic header ingress while embedded RBC-DAG authority is active" + ); + return (AHashSet::new(), Vec::new()); + } + self.add_headers_inner(headers, source) + } + + fn add_headers_inner( + &mut self, + headers: Vec>, + source: DataSource, ) -> (AHashSet, Vec) { let previous_rounds = self.capture_rounds(); + let mut materialization_candidates = headers + .iter() + .map(|header| *header.reference()) + .collect::>(); let (success, missing_parents, processed_refs, processed_blocks) = self.core.add_headers(headers, source); + materialization_candidates.extend(processed_refs.iter().copied()); + self.notify_materialized_shadow_applications(materialization_candidates); if !processed_blocks.is_empty() { // Send blocks to BLS service for verification of embedded BLS fields. self.send_bls_message(BlsServiceMessage::ProcessBlocks(processed_blocks.clone())); @@ -217,11 +270,91 @@ impl Syncer { items: Vec, source: DataSource, ) { + if self.rbc_dag_frontier_authority { + tracing::warn!( + %source, + count = items.len(), + "Rejected generic transaction-data ingress while embedded RBC-DAG authority is active" + ); + return; + } + self.add_transaction_data_inner(items, source); + } + + fn add_transaction_data_inner( + &mut self, + items: Vec, + source: DataSource, + ) { + let references = items + .iter() + .map(|item| item.block_reference) + .collect::>(); self.core.add_transaction_data(items, source); + self.notify_materialized_shadow_applications(references); self.maybe_update_proposal_wait(); self.try_new_block(BlockCreationReason::TransactionData); } + /// Materialize one application header whose authority was established by + /// the carrier actor. Keeping this as a separate core-thread command makes + /// the capability impossible to forge through `BlockBatch::source`. + pub(crate) fn add_authorized_rbc_dag_header( + &mut self, + header: RbcCanonicalHeader, + ) -> (AHashSet, Vec) { + assert!( + self.rbc_dag_frontier_authority, + "carrier-authorized header ingress requires embedded RBC-DAG authority" + ); + let mut block = header.to_authentication_free_block(); + block.preserialize(); + self.add_headers_inner( + vec![Data::new(block)], + DataSource::StarfishRbcDagAuthorizedHeader, + ) + } + + /// Attach payload data already verified against a carrier-authorized + /// canonical header. This cannot be invoked by a peer-controlled source + /// discriminator; only the typed core-thread command exposes it. + pub(crate) fn add_authorized_rbc_dag_payload(&mut self, item: ReconstructedTransactionData) { + assert!( + self.rbc_dag_frontier_authority, + "carrier-authorized payload ingress requires embedded RBC-DAG authority" + ); + self.add_transaction_data_inner(vec![item], DataSource::StarfishRbcDagAuthorizedPayload); + } + + /// Embedded RBC-DAG data availability is proven only by a concrete, + /// data-available DagState block. Payloads may arrive before their header + /// dependencies and remain buffered in Core; any later add path that + /// materializes the block retries this notification using the exact + /// processed references returned by Core. + fn notify_materialized_shadow_applications( + &self, + references: impl IntoIterator, + ) { + let Some(shadow) = self.starfish_rbc_dag_shadow_service.as_ref() else { + return; + }; + for reference in references { + if self.core.dag_state().get_storage_block(reference).is_none() + || !self.core.dag_state().is_data_available(&reference) + { + continue; + } + if let Err(error) = shadow.application_data_available(reference) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + tracing::warn!( + ?reference, + "Failed to record materialized RBC-DAG application availability: {error}" + ); + } + } + } + /// Called after Sailfish RBC certification events have been applied to /// DagState on the core thread. Retries block creation and sequencing /// when any clean vertex is new. @@ -256,7 +389,10 @@ impl Syncer { /// Sequence one exact deterministic carrier-frontier delta. In M7 this is /// the sole application-ordering authority; the legacy Starfish committer /// remains disabled in this mode. - pub fn apply_starfish_rbc_dag_frontier(&mut self, delta: CommittedFrontierDeltaV1) { + pub fn apply_starfish_rbc_dag_frontier( + &mut self, + delta: CommittedFrontierDeltaV1, + ) -> Result { assert!( self.rbc_dag_frontier_authority, "RBC-DAG frontier output requires the explicit authority mode" @@ -266,13 +402,63 @@ impl Syncer { .iter() .map(RbcCanonicalHeader::reference) .collect::>(); - let committed = self.commit_observer.handle_rbc_dag_commit( - self.core.dag_state(), - delta.anchor.carrier(), + match self.core.handle_rbc_dag_committed_delta( + delta.output_sequence, + delta.anchor, &applications, + )? { + RbcDagFrontierApplyOutcome::ExactReplay => Ok(false), + RbcDagFrontierApplyOutcome::Applied(committed) => { + self.commit_observer.handle_rbc_dag_commit(&committed); + self.try_new_block(BlockCreationReason::PostCommit); + Ok(true) + } + } + } + + /// Open the authoritative application-production gate only after the + /// service bridge observes `Ready`. Because the bridge awaits every prior + /// command, this is also a FIFO persistence barrier for recovery output. + pub(crate) fn activate_starfish_rbc_dag_authority(&mut self) { + assert!( + self.rbc_dag_frontier_authority, + "only embedded RBC-DAG authority mode has a startup barrier" ); - self.core.handle_rbc_dag_committed_delta(committed); - self.try_new_block(BlockCreationReason::PostCommit); + if self.rbc_dag_authority_ready { + return; + } + self.rbc_dag_authority_ready = true; + self.core.enable_rbc_dag_application_production(); + let initial_round = self.core.next_block_round(); + self.force_new_block(initial_round); + } + + /// Acknowledge that the exact local application header is durably bound + /// into a carrier. This releases application production independently of + /// later RBC delivery/consensus commitment, preserving the carrier + /// pipeline while bounding producer lead to one header. + pub fn apply_starfish_rbc_dag_application_assigned(&mut self, reference: BlockReference) { + assert!( + self.rbc_dag_frontier_authority, + "RBC-DAG application assignment requires the explicit authority mode" + ); + let Some(expected) = self.rbc_dag_pending_application else { + tracing::debug!( + ?reference, + "Ignoring stale RBC-DAG application-assignment acknowledgement" + ); + return; + }; + if expected != reference { + tracing::debug!( + ?expected, + ?reference, + "Ignoring RBC-DAG application-assignment acknowledgement for a different header" + ); + return; + } + self.rbc_dag_pending_application = None; + self.try_new_block(BlockCreationReason::CertificateEvent); } /// Store a Sailfish++ timeout certificate in DagState and retry block @@ -338,6 +524,9 @@ impl Syncer { /// round can lag the threshold clock) target the round they can /// actually enter. pub fn try_new_block_relaxed(&mut self, proposal_round: RoundNumber) -> bool { + if !self.rbc_dag_authority_ready || self.rbc_dag_pending_application.is_some() { + return false; + } if self.core.dag_state().proposal_round() != proposal_round { return false; } @@ -358,6 +547,9 @@ impl Syncer { } fn try_new_block(&mut self, reason: BlockCreationReason) -> bool { + if !self.rbc_dag_authority_ready || self.rbc_dag_pending_application.is_some() { + return false; + } self.maybe_update_proposal_wait(); if !self.core.committee().is_quorum(self.subscriber_stake) { return false; @@ -374,29 +566,27 @@ impl Syncer { } fn create_new_block(&mut self, reason: BlockCreationReason) -> bool { + if self.rbc_dag_pending_application.is_some() { + return false; + } tracing::debug!("Attempt to create new block in syncer after one trigger"); let previous_rounds = self.capture_rounds(); if let Some(ref block) = self.core.try_new_block(reason.as_str()) { if self.core.dag_state().consensus_protocol.is_starfish_rbc() { let canonical = RbcCanonicalHeader::from_block_header(block.header()) .expect("locally built Starfish-RBC block must have canonical header content"); - let selected = self - .starfish_rbc_service - .as_ref() - .expect("Starfish-RBC protocol must start its RBC service") - .start_local_header_with_payload_blocking( - RbcLocalHeader::from_canonical(&canonical), - block.transaction_data().cloned(), - ) - .expect("local Starfish-RBC header must be accepted before dissemination"); - assert_eq!( - selected.reference(), - *block.reference(), - "RBC service selected a different local header reference" - ); - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.local_header(&canonical) { + if self.rbc_dag_frontier_authority { + self.rbc_dag_pending_application = Some(canonical.reference()); + let shadow = self + .starfish_rbc_dag_shadow_service + .as_ref() + .expect("embedded RBC-DAG authority must start its carrier service"); + if let Err(error) = shadow.local_application( + &canonical, + block.transaction_data().cloned().map(Arc::new), + ) { self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); self.metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["local", "dropped"]) @@ -404,13 +594,35 @@ impl Syncer { tracing::warn!( "Failed to enqueue RBC-DAG application carrier; the research run is invalid: {error}" ); - } else if let Err(error) = - shadow.application_data_available(canonical.reference()) - { - self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - tracing::warn!( - "Failed to record local RBC-DAG application availability: {error}" - ); + } else { + self.notify_materialized_shadow_applications([canonical.reference()]); + } + } else { + let selected = self + .starfish_rbc_service + .as_ref() + .expect("direct Starfish-RBC mode must start its RBC service") + .start_local_header_with_payload_blocking( + RbcLocalHeader::from_canonical(&canonical), + block.transaction_data().cloned(), + ) + .expect("local Starfish-RBC header must be accepted before dissemination"); + assert_eq!( + selected.reference(), + *block.reference(), + "RBC service selected a different local header reference" + ); + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.local_header(&canonical) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); + self.metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["local", "dropped"]) + .inc(); + tracing::warn!("Failed to enqueue RBC-DAG mirror carrier: {error}"); + } else { + self.notify_materialized_shadow_applications([canonical.reference()]); + } } } } @@ -565,6 +777,8 @@ impl SyncerSignals for bool { #[cfg(test)] mod tests { + use std::time::Duration; + use prometheus::Registry; use tempfile::TempDir; @@ -573,10 +787,21 @@ mod tests { block_handler::BlockHandler, committee::Committee, config::{DisseminationMode, NodePrivateConfig, StorageBackend}, - crypto::Signer, + crypto::{Signer, TransactionsCommitment, mac_keyrings_for_test}, dag_state::{ConsensusProtocol, DagState}, + encoder::ShardEncoder, metrics::Metrics, - types::BaseTransaction, + starfish_rbc_dag::{ + RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, + storage::ShadowWalSyncPolicyV1, + }, + starfish_rbc_dag_shadow::ShadowAuthorizerV1, + starfish_rbc_dag_shadow_service::{ + ShadowServiceEventV1, start_starfish_rbc_dag_autonomous_clock_service_v1, + }, + types::{ + BaseTransaction, BlockAuthenticationScheme, Encoder, Transaction, TransactionData, + }, }; #[derive(Default)] @@ -602,14 +827,7 @@ mod tests { Vec::new() } - fn handle_rbc_dag_commit( - &mut self, - _dag_state: &DagState, - _anchor: BlockReference, - _applications: &[BlockReference], - ) -> Vec { - Vec::new() - } + fn handle_rbc_dag_commit(&mut self, _committed: &[CommittedSubDag]) {} fn recover_committed( &mut self, @@ -852,6 +1070,319 @@ mod tests { Data::new(block) } + fn make_starfish_rbc_round_1_application( + committee: &Committee, + authority: AuthorityIndex, + receiver: AuthorityIndex, + ) -> ( + Data, + RbcCanonicalHeader, + ReconstructedTransactionData, + ) { + let payload_byte = u8::try_from(authority).unwrap(); + let transactions = vec![BaseTransaction::Share(Transaction::new(vec![ + payload_byte; + 64 + ]))]; + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded = encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + let mut block = VerifiedBlock::new_starfish_rbc( + authority, + 1, + committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 0)) + .collect(), + Vec::new(), + u64::from(authority) + 1, + transactions.clone(), + Some(encoded.clone()), + ); + let canonical = RbcCanonicalHeader::from_block_header(block.header()).unwrap(); + block.preserialize(); + + let mut transaction_data = TransactionData::new(transactions); + transaction_data.preserialize(); + let (commitment, proof) = + TransactionsCommitment::new_from_encoded_transactions(&encoded, receiver as usize); + assert_eq!(commitment, canonical.transactions_commitment()); + let mut shard_data = ProvableShard::new( + encoded[receiver as usize].clone(), + receiver as usize, + proof, + commitment, + ); + shard_data.preserialize(); + let payload = ReconstructedTransactionData { + block_reference: canonical.reference(), + transaction_data, + shard_data, + }; + (Data::new(block), canonical, payload) + } + + fn make_reconstructed_payload( + full_block: &Data, + committee: &Committee, + receiver: AuthorityIndex, + ) -> ReconstructedTransactionData { + let transactions = full_block + .transaction_data() + .expect("test application must carry transaction data") + .transactions() + .clone(); + let mut encoder = Encoder::new(2, 4, 2).unwrap(); + let encoded = encoder.encode_transactions( + &transactions, + committee.info_length(), + committee.len() - committee.info_length(), + ); + let mut transaction_data = TransactionData::new(transactions); + transaction_data.preserialize(); + let (commitment, proof) = + TransactionsCommitment::new_from_encoded_transactions(&encoded, receiver as usize); + let mut shard_data = ProvableShard::new( + encoded[receiver as usize].clone(), + receiver as usize, + proof, + commitment, + ); + shard_data.preserialize(); + ReconstructedTransactionData { + block_reference: *full_block.reference(), + transaction_data, + shard_data, + } + } + + async fn wait_for_shadow_ready(events: &mut mpsc::Receiver) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match events.recv().await { + Some(ShadowServiceEventV1::Ready { autonomous_clock }) => { + assert!(autonomous_clock); + break; + } + Some(_) => {} + None => panic!("autonomous shadow service stopped before Ready"), + } + } + }) + .await + .expect("autonomous shadow service did not become ready"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn rbc_dag_gate_and_typed_ingress_are_enforced() { + let authority = 0; + let committee = Committee::new_for_benchmarks(4); + let registry = Registry::new(); + let (metrics, _reporter) = Metrics::new( + ®istry, + Some(committee.as_ref()), + Some("starfish-rbc"), + None, + ); + let dir = TempDir::new().unwrap(); + let recovered = DagState::open( + authority, + dir.path(), + metrics.clone(), + committee.clone(), + "honest".to_string(), + "starfish-rbc".to_string(), + &StorageBackend::Rocksdb, + false, + DisseminationMode::ProtocolDefault, + ); + let private_config = NodePrivateConfig::new_for_tests(authority); + let (core, _) = Core::open( + TestBlockHandler, + authority, + committee.clone(), + private_config, + metrics.clone(), + recovered, + None, + ); + + let shadow_directory = TempDir::new().unwrap(); + let keyrings = mac_keyrings_for_test(committee.len()); + let committee_context = RbcDagCommitteeContextV1::new(committee.clone()).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0x53; 32]).unwrap(), + &committee_context, + BlockAuthenticationScheme::MacVector, + ); + let (shadow_service, mut shadow_events, shadow_task) = + start_starfish_rbc_dag_autonomous_clock_service_v1( + shadow_directory.path().join("standalone.wal"), + committee_context, + authority, + context, + ShadowAuthorizerV1::MacVector(keyrings[authority as usize].clone()), + Vec::new(), + Duration::from_secs(3_600), + ShadowWalSyncPolicyV1::EveryBatch, + ) + .unwrap(); + wait_for_shadow_ready(&mut shadow_events).await; + // This unit drives the Syncer boundary directly instead of running the + // production event bridge. Keep draining the actor's bounded event + // channel so local carrier fanout cannot block the actor (and hence a + // later graceful shutdown) behind events this test intentionally does + // not consume. + let shadow_event_drain = + tokio::spawn(async move { while shadow_events.recv().await.is_some() {} }); + + let mut syncer = Syncer::new( + core, + TestSignals::default(), + NoopCommitObserver, + metrics, + None, + None, + None, + Some(shadow_service.clone()), + true, + ); + syncer.connected_authorities.extend([1, 2, 3]); + syncer.subscribed_by_authorities.extend([1, 2, 3]); + syncer.recompute_subscriber_stake(); + + // Production is intentionally closed until the ordered service bridge + // consumes `Ready`. Exercise that authority barrier before asserting + // the one-outstanding application gate. + syncer.activate_starfish_rbc_dag_authority(); + let first = syncer + .rbc_dag_pending_application + .expect("round-one application must wait for carrier assignment"); + assert_eq!(first.round, 1); + assert_eq!(syncer.core.last_proposed(), 1); + assert_eq!(syncer.signals.new_block_ready_count, 1); + assert!(!shadow_task.is_finished()); + + // Peer-controlled generic ingress cannot materialize authentication- + // free applications in standalone mode. + let (full_one, canonical_one, payload_one) = + make_starfish_rbc_round_1_application(&committee, 1, authority); + let reference_one = canonical_one.reference(); + let mut header_one = canonical_one.to_authentication_free_block(); + header_one.preserialize(); + let generic_headers = syncer.add_headers( + vec![Data::new(header_one)], + DataSource::BlockBundleStreamingHeader, + ); + assert!(generic_headers.0.is_empty()); + assert!(generic_headers.1.is_empty()); + assert!( + syncer + .core + .dag_state() + .get_storage_block(reference_one) + .is_none() + ); + + let (full_two, canonical_two, _) = + make_starfish_rbc_round_1_application(&committee, 2, authority); + let reference_two = canonical_two.reference(); + let generic_blocks = + syncer.add_blocks(vec![(full_two, None)], DataSource::BlockBundleStreaming); + assert!(generic_blocks.0.is_empty()); + assert!(generic_blocks.1.is_empty()); + assert!(generic_blocks.2.is_empty()); + assert!( + syncer + .core + .dag_state() + .get_storage_block(reference_two) + .is_none() + ); + + // The actor-only typed header command admits the exact canonical + // application, while the generic payload command remains closed. + let (missing_one, _) = syncer.add_authorized_rbc_dag_header(canonical_one); + assert!(missing_one.is_empty()); + assert!( + syncer + .core + .dag_state() + .get_storage_block(reference_one) + .is_some() + ); + assert!(!syncer.core.dag_state().is_data_available(&reference_one)); + + syncer.add_transaction_data( + vec![payload_one], + DataSource::StarfishRbcDagAuthorizedPayload, + ); + assert!(!syncer.core.dag_state().is_data_available(&reference_one)); + + syncer.add_authorized_rbc_dag_payload(make_reconstructed_payload( + &full_one, &committee, authority, + )); + assert!(syncer.core.dag_state().is_data_available(&reference_one)); + + let (missing_two, _) = syncer.add_authorized_rbc_dag_header(canonical_two); + assert!(missing_two.is_empty()); + assert!( + syncer + .core + .dag_state() + .get_storage_block(reference_two) + .is_some() + ); + + // Even after a typed header quorum advances the application clock, + // all creation triggers remain closed while the first header is pending. + assert_eq!(syncer.core.dag_state().threshold_clock_round(), 2); + assert!(!syncer.try_new_block(BlockCreationReason::NewHeaders)); + assert_eq!(syncer.core.last_proposed(), 1); + assert_eq!(syncer.rbc_dag_pending_application, Some(first)); + + let wrong = BlockReference::new_test(3, first.round); + syncer.apply_starfish_rbc_dag_application_assigned(wrong); + assert_eq!(syncer.rbc_dag_pending_application, Some(first)); + assert_eq!(syncer.core.last_proposed(), 1); + + // The exact acknowledgement releases one proposal attempt. That + // attempt creates round two and immediately closes the gate around + // the new outstanding header; it cannot create more than one block. + syncer.apply_starfish_rbc_dag_application_assigned(first); + let second = syncer + .rbc_dag_pending_application + .expect("round-two application must become the sole outstanding header"); + assert_eq!(second.round, 2); + assert_ne!(second, first); + assert_eq!(syncer.core.last_proposed(), 2); + assert_eq!(syncer.signals.new_block_ready_count, 2); + assert!(!syncer.try_new_block(BlockCreationReason::CertificateEvent)); + assert_eq!(syncer.core.last_proposed(), 2); + + // A duplicate acknowledgement for the old header is stale and must + // neither release nor replace the current gate. + syncer.apply_starfish_rbc_dag_application_assigned(first); + assert_eq!(syncer.rbc_dag_pending_application, Some(second)); + assert_eq!(syncer.core.last_proposed(), 2); + + // With no round-three quorum, the exact second acknowledgement simply + // clears the gate. A later duplicate is harmless and leaves it clear. + syncer.apply_starfish_rbc_dag_application_assigned(second); + assert_eq!(syncer.rbc_dag_pending_application, None); + assert_eq!(syncer.core.last_proposed(), 2); + syncer.apply_starfish_rbc_dag_application_assigned(second); + assert_eq!(syncer.rbc_dag_pending_application, None); + assert_eq!(syncer.signals.new_block_ready_count, 2); + + shadow_service.shutdown().await.unwrap(); + shadow_task.await.unwrap(); + shadow_event_drain.await.unwrap(); + } + #[test] fn normal_block_creation_uses_next_missing_round() { let mut syncer = open_test_syncer_with_future_rounds(); diff --git a/crates/starfish-core/src/tidehunter_store.rs b/crates/starfish-core/src/tidehunter_store.rs index 1339b669..e9b6c35d 100644 --- a/crates/starfish-core/src/tidehunter_store.rs +++ b/crates/starfish-core/src/tidehunter_store.rs @@ -15,7 +15,7 @@ use tidehunter::{ use crate::{ dag_state::CommitData, data::Data, - store::Store, + store::{RbcDagFrontierReceipt, Store, validate_rbc_dag_frontier_commit_batch}, types::{ BlockHeader, BlockReference, ProvableShard, RoundNumber, TransactionData, VerifiedBlock, }, @@ -31,6 +31,7 @@ const PREFIX_LEN: usize = 3; /// Number of mutexes per key space for concurrency control. Must be power of 2. const MUTEXES: usize = 64; +const LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY: [u8; KEY_SIZE] = [0; KEY_SIZE]; pub struct TideHunterStore { db: Arc, @@ -40,6 +41,7 @@ pub struct TideHunterStore { ks_shard_data: KeySpace, ks_commits: KeySpace, ks_dual_dag_clean: KeySpace, + ks_rbc_dag_frontier_receipt: KeySpace, } impl TideHunterStore { @@ -83,6 +85,7 @@ impl TideHunterStore { let ks_shard_data = Self::add_ks(&mut builder, "shard_data"); let ks_commits = Self::add_ks(&mut builder, "commits"); let ks_dual_dag_clean = Self::add_ks(&mut builder, "sailfish_certified"); + let ks_rbc_dag_frontier_receipt = Self::add_ks(&mut builder, "rbc_dag_frontier_receipt"); let key_shape = builder.build(); let config = Arc::new(Config { @@ -105,6 +108,7 @@ impl TideHunterStore { ks_shard_data, ks_commits, ks_dual_dag_clean, + ks_rbc_dag_frontier_receipt, }) } @@ -247,6 +251,36 @@ impl Store for TideHunterStore { .map_err(|e| io::Error::other(format!("TideHunter commit batch: {e:?}"))) } + fn store_commits_with_rbc_dag_receipt( + &self, + committed_sub_dags: Vec, + receipt: RbcDagFrontierReceipt, + ) -> io::Result<()> { + validate_rbc_dag_frontier_commit_batch(&committed_sub_dags, &receipt)?; + let receipt_bytes = receipt.to_bytes()?; + + let mut batch = self.db.write_batch(); + if committed_sub_dags.is_empty() { + batch.delete( + self.ks_commits, + Self::encode_key(&receipt.carrier_anchor).to_vec(), + ); + } else { + let commit_data = &committed_sub_dags[0]; + let key = Self::encode_key(&commit_data.leader); + let value = bincode::serialize(&commit_data).map_err(io::Error::other)?; + batch.write(self.ks_commits, key.to_vec(), value); + } + batch.write( + self.ks_rbc_dag_frontier_receipt, + LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY.to_vec(), + receipt_bytes, + ); + batch.commit().map_err(|e| { + io::Error::other(format!("TideHunter RBC-DAG commit/receipt batch: {e:?}")) + }) + } + fn get_commit(&self, reference: &BlockReference) -> io::Result> { let key = Self::encode_key(reference); match self @@ -263,6 +297,20 @@ impl Store for TideHunterStore { } } + fn read_latest_rbc_dag_frontier_receipt(&self) -> io::Result> { + match self + .db + .get( + self.ks_rbc_dag_frontier_receipt, + &LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY, + ) + .map_err(|e| io::Error::other(format!("TideHunter get receipt: {e:?}")))? + { + Some(bytes) => RbcDagFrontierReceipt::from_bytes(&bytes).map(Some), + None => Ok(None), + } + } + fn store_header_bytes(&self, reference: &BlockReference, bytes: &[u8]) -> io::Result<()> { let key = Self::encode_key(reference); self.db @@ -404,8 +452,32 @@ impl Store for TideHunterStore { #[cfg(test)] mod tests { + use tempfile::TempDir; + use super::TideHunterStore; - use crate::types::BlockReference; + use crate::{ + dag_state::CommitData, + store::{RbcDagFrontierReceipt, Store}, + types::{BlockReference, MAX_COMMITTEE_SIZE}, + }; + + fn commit(leader: BlockReference, committed_rounds: Vec) -> CommitData { + CommitData { + leader, + sub_dag: vec![BlockReference::new_test(1, leader.round)], + committed_rounds, + } + } + + fn assert_commit(store: &impl Store, expected: &CommitData) { + let actual = store + .get_commit(&expected.leader) + .expect("commit read should succeed") + .expect("commit should exist"); + assert_eq!(actual.leader, expected.leader); + assert_eq!(actual.sub_dag, expected.sub_dag); + assert_eq!(actual.committed_rounds, expected.committed_rounds); + } #[test] fn encode_key_preserves_u16_authority() { @@ -429,4 +501,129 @@ mod tests { assert_eq!(&low_key[4..6], &255u16.to_be_bytes()); assert_eq!(&high_key[4..6], &256u16.to_be_bytes()); } + + #[test] + fn rbc_dag_receipt_and_commits_are_atomic_and_latest_is_a_point_value() { + let temp_dir = TempDir::new().unwrap(); + let store = TideHunterStore::open(temp_dir.path()).unwrap(); + + let legacy_leader = BlockReference::new_test(2, 253); + let legacy_commit = commit(legacy_leader, vec![253; 4]); + store.store_commits(vec![legacy_commit.clone()]).unwrap(); + assert_commit(&store, &legacy_commit); + assert!( + store + .read_latest_rbc_dag_frontier_receipt() + .unwrap() + .is_none() + ); + + // A control-only frontier has no new application commits, but its + // durable cursor must still advance. + let first_anchor = BlockReference::new_test(7, 255); + let first_receipt = RbcDagFrontierReceipt { + carrier_anchor: first_anchor, + output_sequence: 255, + committed_rounds: vec![250, 251, 252, 253], + }; + let stale_first_commit = commit(first_anchor, first_receipt.committed_rounds.clone()); + store.store_commits(vec![stale_first_commit]).unwrap(); + assert!(store.get_commit(&first_anchor).unwrap().is_some()); + store + .store_commits_with_rbc_dag_receipt(Vec::new(), first_receipt.clone()) + .unwrap(); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(first_receipt) + ); + assert!(store.get_commit(&first_anchor).unwrap().is_none()); + + // The exact application commit is stored under the consensus carrier + // anchor so Core can reconstruct the compact receipt's application + // references after restart. + let second_anchor = BlockReference::new_test(7, 256); + let application_commit = commit(second_anchor, vec![255, 256, 255, 256]); + let second_receipt = RbcDagFrontierReceipt { + carrier_anchor: second_anchor, + output_sequence: 256, + committed_rounds: vec![255, 256, 255, 256], + }; + store + .store_commits_with_rbc_dag_receipt( + vec![application_commit.clone()], + second_receipt.clone(), + ) + .unwrap(); + assert_commit(&store, &application_commit); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(second_receipt.clone()) + ); + + // Mismatched/multiple application commits are rejected before either + // commit data or the latest receipt can change. + let mismatched = commit(BlockReference::new_test(2, 254), vec![255, 256, 255, 256]); + let mismatched_watermarks = commit(second_anchor, vec![1; 4]); + for invalid in [ + vec![mismatched], + vec![mismatched_watermarks], + vec![application_commit.clone(), application_commit.clone()], + ] { + let error = store + .store_commits_with_rbc_dag_receipt(invalid, second_receipt.clone()) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(second_receipt.clone()) + ); + } + + // Reusing the exact anchor for a control-only marker atomically + // removes stale application CommitData, preserving absence semantics. + let control_receipt = RbcDagFrontierReceipt { + carrier_anchor: second_anchor, + output_sequence: 257, + committed_rounds: second_receipt.committed_rounds.clone(), + }; + store + .store_commits_with_rbc_dag_receipt(Vec::new(), control_receipt.clone()) + .unwrap(); + assert!(store.get_commit(&second_anchor).unwrap().is_none()); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(control_receipt.clone()) + ); + + // Receipt validation happens before the batch is submitted, so an + // invalid vector cannot partially write its application commit or + // replace the last valid cursor. + let rejected_leader = BlockReference::new_test(3, 257); + let rejected = commit(rejected_leader, vec![257; 4]); + for committed_rounds in [Vec::new(), vec![0; usize::from(MAX_COMMITTEE_SIZE) + 1]] { + let invalid = RbcDagFrontierReceipt { + carrier_anchor: BlockReference::new_test(7, 257), + output_sequence: 258, + committed_rounds, + }; + let error = store + .store_commits_with_rbc_dag_receipt(vec![rejected.clone()], invalid) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + } + assert!(store.get_commit(&rejected_leader).unwrap().is_none()); + assert_eq!( + store.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(control_receipt.clone()) + ); + + drop(store); + let reopened = TideHunterStore::open(temp_dir.path()).unwrap(); + assert_commit(&reopened, &legacy_commit); + assert!(reopened.get_commit(&second_anchor).unwrap().is_none()); + assert_eq!( + reopened.read_latest_rbc_dag_frontier_receipt().unwrap(), + Some(control_receipt) + ); + } } diff --git a/crates/starfish-core/src/transactions_generator.rs b/crates/starfish-core/src/transactions_generator.rs index 0ca474f9..285c316e 100644 --- a/crates/starfish-core/src/transactions_generator.rs +++ b/crates/starfish-core/src/transactions_generator.rs @@ -5,16 +5,19 @@ use std::{ cmp::min, sync::{Arc, atomic::Ordering}, - time::{Duration, Instant}, + time::Duration, }; use rand::{Rng, RngCore, SeedableRng, rngs::StdRng}; -use tokio::sync::mpsc; +use tokio::{ + sync::{mpsc, watch}, + time::{Instant, MissedTickBehavior, interval_at, sleep_until}, +}; use crate::{ config::{NodePublicConfig, Parameters, TransactionMode}, crypto::AsBytes, - metrics::Metrics, + metrics::{BenchmarkGeneratorState, BenchmarkTransactionWindow, Metrics}, runtime::{self, timestamp_utc}, types::{AuthorityIndex, Transaction}, }; @@ -25,6 +28,14 @@ pub struct TransactionGenerator { parameters: Parameters, node_public_config: NodePublicConfig, metrics: Arc, + start_gate: Option>>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SubmissionOutcome { + Submitted, + Cutoff, + Closed, } impl TransactionGenerator { @@ -39,14 +50,73 @@ impl TransactionGenerator { transactions } + async fn submit_before_end( + &self, + block: Vec, + generation_end: Option, + ) -> SubmissionOutcome { + let Some(end) = generation_end else { + return if self.sender.send(block).await.is_ok() { + SubmissionOutcome::Submitted + } else { + SubmissionOutcome::Closed + }; + }; + if Instant::now() >= end { + return SubmissionOutcome::Cutoff; + } + // Deadline first makes t == end exclusive even when channel capacity + // and the cutoff become ready in the same scheduler turn. A pending + // send is cancelled without publishing its block, so it cannot become + // post-window offered work or drain debt. + tokio::select! { + biased; + _ = sleep_until(end) => SubmissionOutcome::Cutoff, + result = self.sender.send(block) => { + if result.is_ok() { + SubmissionOutcome::Submitted + } else { + SubmissionOutcome::Closed + } + } + } + } + + #[cfg(test)] pub fn start( sender: mpsc::Sender>, seed: AuthorityIndex, parameters: Parameters, node_public_config: NodePublicConfig, metrics: Arc, + ) { + Self::start_with_gate(sender, seed, parameters, node_public_config, metrics, None); + } + + pub fn start_with_gate( + sender: mpsc::Sender>, + seed: AuthorityIndex, + parameters: Parameters, + node_public_config: NodePublicConfig, + metrics: Arc, + start_gate: Option>>, ) { assert!(parameters.transaction_size > 8 + 8); // 8 bytes timestamp + 8 bytes random + // Publish the finite-window warmup state synchronously. The spawned + // task may not be polled until after the benchmark harness inspects + // readiness, so setting this only inside `run` creates a false active + // window and can baseline before the topology is complete. + if parameters.benchmark_duration.is_some() { + metrics.metrics_active.store(false, Ordering::Relaxed); + metrics + .transaction_metrics_active + .store(false, Ordering::Relaxed); + } + if start_gate.is_some() { + metrics + .benchmark_generator_state + .store(BenchmarkGeneratorState::Waiting as u8, Ordering::Release); + } runtime::Handle::current().spawn( Self { sender, @@ -54,6 +124,7 @@ impl TransactionGenerator { parameters, node_public_config, metrics, + start_gate, } .run(), ); @@ -62,9 +133,13 @@ impl TransactionGenerator { pub async fn run(mut self) { let load = self.parameters.load; let max_transactions_per_block_interval = load.div_ceil(Self::BATCHES_IN_SECOND); + let coordinated_start = self.start_gate.is_some(); // Add a small extra delay proportional to committee size to give - // connections time to come up. Calibrated so n=100 lands at ~15s - // total when initial_delay is the 10s default. + // connections time to come up in normal production/ungated runs. + // Calibrated so n=100 lands at ~15s total when initial_delay is the + // 10s default. A coordinated benchmark gate is released only after + // topology and clock activation, so it deliberately skips this + // second warmup and starts the shared finite window immediately. let initial_delay_plus_extra_delay = self.parameters.initial_delay + Duration::from_millis( (self.node_public_config.identifiers.len() as f64 / 100.0 * 5000.0) as u64, @@ -72,24 +147,30 @@ impl TransactionGenerator { let benchmark_duration = self.parameters.benchmark_duration; // When the orchestrator sets a finite benchmark window, gate metrics - // off during the warmup so the validator's `benchmark_duration` - // counter only ticks inside the active submission window. Without - // this, the warmup seconds inflate the TPS denominator. - if benchmark_duration.is_some() { - self.metrics.metrics_active.store(false, Ordering::Relaxed); + // off while awaiting either the coordinated release or the normal + // warmup so `benchmark_duration` counts only active submissions. + let finite_window_description = match benchmark_duration { + Some(d) => format!(", stopping after {} sec of generation", d.as_secs()), + None => String::new(), + }; + if coordinated_start { + tracing::info!( + "Starting tx generator behind the coordinated release gate; \ + targeting {load} tx/s immediately after release \ + (up to {max_transactions_per_block_interval} transactions every {} ms){}", + Self::TARGET_BLOCK_INTERVAL.as_millis(), + finite_window_description, + ); + } else { + tracing::info!( + "Starting tx generator. After {} sec, \ + targeting {load} tx/s \ + (up to {max_transactions_per_block_interval} transactions every {} ms){}", + initial_delay_plus_extra_delay.as_secs(), + Self::TARGET_BLOCK_INTERVAL.as_millis(), + finite_window_description, + ); } - - tracing::info!( - "Starting tx generator. After {} sec, \ - targeting {load} tx/s \ - (up to {max_transactions_per_block_interval} transactions every {} ms){}", - initial_delay_plus_extra_delay.as_secs(), - Self::TARGET_BLOCK_INTERVAL.as_millis(), - match benchmark_duration { - Some(d) => format!(", stopping after {} sec of generation", d.as_secs()), - None => String::new(), - }, - ); let max_block_size = self.node_public_config.parameters.max_block_size; let target_block_size = min(max_block_size, max_transactions_per_block_interval); @@ -101,46 +182,69 @@ impl TransactionGenerator { }); let mut counter: u64 = 0; - let mut tx_to_report = 0; let mut random: u64 = self.rng.gen(); let mut load_carry = 0; // Pre-allocated payload buffer reused in AllZero mode. let zeros = vec![0u8; tx_size - 8 - 8]; - let mut interval = runtime::TimeInterval::new(Self::TARGET_BLOCK_INTERVAL); - runtime::sleep(initial_delay_plus_extra_delay).await; - let generation_start = Instant::now(); + let coordinated_window = if let Some(mut start_gate) = self.start_gate.take() { + match start_gate.wait_for(Option::is_some).await { + Ok(window) => *window, + Err(_) => { + self.fail_coordinated_window(); + return; + } + } + } else { + None + }; + + if !coordinated_start { + runtime::sleep(initial_delay_plus_extra_delay).await; + } + + let generation_start = coordinated_window + .map(|window| window.start) + .unwrap_or_else(Instant::now); + let generation_end = coordinated_window.map(|window| window.end).or_else(|| { + benchmark_duration.and_then(|duration| generation_start.checked_add(duration)) + }); + if Instant::now() < generation_start { + sleep_until(generation_start).await; + } + + // Anchor every coordinated generator to the same absolute tick grid. + // MissedTickBehavior::Skip prevents a delayed task from bursting old + // batches, and comparing the scheduled tick strictly with `end` + // excludes the historical extra batch at t == duration. + let mut interval = interval_at(generation_start, Self::TARGET_BLOCK_INTERVAL); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); // Open the active metrics window: anchor `benchmark_duration`'s // clock to this instant so the TPS denominator only counts seconds // during which transactions are actually being submitted. - let active_start_micros = self - .metrics - .validator_start - .elapsed() + let active_start_micros = generation_start + .saturating_duration_since(self.metrics.validator_start) .as_micros() .min(u64::MAX as u128) as u64; self.metrics .active_start_micros .store(active_start_micros, Ordering::Relaxed); self.metrics.metrics_active.store(true, Ordering::Relaxed); + self.metrics + .transaction_metrics_active + .store(true, Ordering::Relaxed); + if coordinated_start { + self.metrics + .benchmark_generator_state + .store(BenchmarkGeneratorState::Active as u8, Ordering::Release); + } - loop { - if let Some(limit) = benchmark_duration { - if generation_start.elapsed() >= limit { - tracing::info!( - "Tx generator reached benchmark duration ({} sec); \ - stopping submissions and closing the metrics window.", - limit.as_secs(), - ); - // Close the active metrics window so commits arriving - // during the wind-down don't pollute cumulative - // quantiles or skew the TPS denominator. - self.metrics.metrics_active.store(false, Ordering::Relaxed); - break; - } + 'generation: loop { + let scheduled_tick = interval.tick().await; + if generation_end.is_some_and(|end| scheduled_tick >= end) { + break; } - interval.tick().await; let timestamp = (timestamp_utc().as_millis() as u64).to_le_bytes(); let transactions_per_block_interval = Self::transactions_for_interval(load, &mut load_carry); @@ -167,29 +271,74 @@ impl TransactionGenerator { block.push(Transaction::new(transaction)); block_size += tx_size; counter += 1; - tx_to_report += 1; if block_size >= max_block_size { - if self.sender.send(block.clone()).await.is_err() { - return; + let submitted = block.len() as u64; + match self.submit_before_end(block.clone(), generation_end).await { + SubmissionOutcome::Submitted => {} + SubmissionOutcome::Cutoff => break 'generation, + SubmissionOutcome::Closed => { + self.fail_coordinated_window(); + return; + } } + self.metrics.submitted_transactions.inc_by(submitted); + self.metrics + .submitted_transactions_bytes + .inc_by(submitted.saturating_mul(tx_size as u64)); block.clear(); block_size = 0; } } tracing::debug!("Generator send {} transactions", block.len()); - if !block.is_empty() && self.sender.send(block).await.is_err() { - return; - } - - if counter.is_multiple_of(10_000) { + if !block.is_empty() { + let submitted = block.len() as u64; + match self.submit_before_end(block, generation_end).await { + SubmissionOutcome::Submitted => {} + SubmissionOutcome::Cutoff => break 'generation, + SubmissionOutcome::Closed => { + self.fail_coordinated_window(); + return; + } + } + self.metrics.submitted_transactions.inc_by(submitted); self.metrics .submitted_transactions_bytes - .inc_by(tx_to_report * tx_size as u64); - self.metrics.submitted_transactions.inc_by(tx_to_report); - tx_to_report = 0 + .inc_by(submitted.saturating_mul(tx_size as u64)); } } + + self.metrics.metrics_active.store(false, Ordering::Release); + if coordinated_start { + // The local benchmark keeps transaction observation open during + // its bounded drain and closes it after every offered transaction + // is observed (or reports an explicit incomplete drain). + self.metrics + .benchmark_generator_state + .store(BenchmarkGeneratorState::Finished as u8, Ordering::Release); + } else { + self.metrics + .transaction_metrics_active + .store(false, Ordering::Release); + } + } + + fn fail_coordinated_window(&self) { + self.metrics.metrics_active.store(false, Ordering::Release); + self.metrics + .transaction_metrics_active + .store(false, Ordering::Release); + if self.start_gate.is_some() + || BenchmarkGeneratorState::from_u8( + self.metrics + .benchmark_generator_state + .load(Ordering::Acquire), + ) != BenchmarkGeneratorState::Disabled + { + self.metrics + .benchmark_generator_state + .store(BenchmarkGeneratorState::Failed as u8, Ordering::Release); + } } pub fn extract_timestamp(transaction: &Transaction) -> Duration { @@ -202,7 +351,23 @@ impl TransactionGenerator { #[cfg(test)] mod tests { + use std::{ + net::{IpAddr, Ipv4Addr}, + sync::atomic::Ordering, + time::Duration, + }; + + use prometheus::Registry; + use tokio::{ + sync::{mpsc, watch}, + time::{Instant, timeout}, + }; + use super::TransactionGenerator; + use crate::{ + config::{NodePublicConfig, Parameters}, + metrics::{BenchmarkGeneratorState, BenchmarkTransactionWindow, Metrics}, + }; #[test] fn transactions_for_interval_matches_target_rate() { @@ -216,4 +381,175 @@ mod tests { assert_eq!(carry, 0, "load={load}"); } } + + #[tokio::test] + async fn ungated_finite_generator_keeps_metrics_closed_during_warmup() { + let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); + metrics.metrics_active.store(true, Ordering::Relaxed); + let (sender, mut receiver) = mpsc::channel(1); + let mut parameters = Parameters::almost_default(1); + parameters.benchmark_duration = Some(Duration::from_secs(1)); + parameters.initial_delay = Duration::from_secs(60); + let public_config = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); + + TransactionGenerator::start(sender, 0, parameters, public_config, metrics.clone()); + + assert!(!metrics.metrics_active.load(Ordering::Relaxed)); + assert!( + timeout(Duration::from_millis(100), receiver.recv()) + .await + .is_err(), + "ungated generator skipped its configured warmup" + ); + assert!(!metrics.metrics_active.load(Ordering::Relaxed)); + } + + #[tokio::test] + async fn coordinated_generator_starts_immediately_after_release() { + let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); + let (sender, mut receiver) = mpsc::channel(1); + let mut parameters = Parameters::almost_default(20); + parameters.benchmark_duration = Some(Duration::from_secs(1)); + parameters.initial_delay = Duration::from_secs(60); + let public_config = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); + let (release, gate) = watch::channel(None); + + TransactionGenerator::start_with_gate( + sender, + 0, + parameters, + public_config, + metrics.clone(), + Some(gate), + ); + assert!( + timeout(Duration::from_millis(100), receiver.recv()) + .await + .is_err(), + "generator submitted before the coordinated release" + ); + assert!(!metrics.metrics_active.load(Ordering::Relaxed)); + + let start = Instant::now() + Duration::from_millis(10); + release.send_replace(BenchmarkTransactionWindow::new( + start, + start + Duration::from_secs(1), + )); + timeout(Duration::from_millis(500), receiver.recv()) + .await + .expect("released generator waited for the configured initial delay") + .expect("released generator channel closed"); + assert!(metrics.metrics_active.load(Ordering::Relaxed)); + + assert!( + timeout(Duration::from_millis(20), receiver.recv()) + .await + .is_err(), + "released generator emitted a stale-interval catch-up burst" + ); + timeout(Duration::from_millis(250), receiver.recv()) + .await + .expect("released generator did not establish a fresh cadence") + .expect("released generator channel closed"); + } + + #[tokio::test] + async fn coordinated_window_counts_every_send_and_excludes_end_tick() { + let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); + let (sender, mut receiver) = mpsc::channel(16); + let mut parameters = Parameters::almost_default(20); + parameters.benchmark_duration = Some(Duration::from_secs(60)); + parameters.initial_delay = Duration::from_secs(60); + let public_config = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); + let (release, gate) = watch::channel(None); + + TransactionGenerator::start_with_gate( + sender, + 0, + parameters, + public_config, + metrics.clone(), + Some(gate), + ); + assert_eq!( + BenchmarkGeneratorState::from_u8( + metrics.benchmark_generator_state.load(Ordering::Acquire) + ), + BenchmarkGeneratorState::Waiting + ); + assert_eq!(metrics.submitted_transactions.get(), 0); + + let start = Instant::now() + Duration::from_millis(20); + let end = start + Duration::from_millis(200); + release.send_replace(BenchmarkTransactionWindow::new(start, end)); + timeout(Duration::from_secs(1), async { + loop { + if BenchmarkGeneratorState::from_u8( + metrics.benchmark_generator_state.load(Ordering::Acquire), + ) == BenchmarkGeneratorState::Finished + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("coordinated generator did not finish its absolute window"); + + let mut received = 0usize; + while let Ok(block) = receiver.try_recv() { + received += block.len(); + } + assert_eq!(received, 4, "ticks must be start, +50, +100, +150 ms"); + assert_eq!(metrics.submitted_transactions.get(), 4); + assert_eq!(metrics.submitted_transactions_bytes.get(), 4 * 512); + assert!(!metrics.metrics_active.load(Ordering::Acquire)); + assert!(metrics.transaction_metrics_active.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn coordinated_window_cancels_a_backpressured_send_at_cutoff() { + let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); + // The first tick fills this channel. The next scheduled pre-end send + // must remain pending until the common cutoff and then be cancelled, + // not counted as offered work after the window. + let (sender, mut receiver) = mpsc::channel(1); + let mut parameters = Parameters::almost_default(20); + parameters.benchmark_duration = Some(Duration::from_secs(60)); + let public_config = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); + let (release, gate) = watch::channel(None); + TransactionGenerator::start_with_gate( + sender, + 0, + parameters, + public_config, + metrics.clone(), + Some(gate), + ); + + let start = Instant::now() + Duration::from_millis(20); + let end = start + Duration::from_millis(120); + release.send_replace(BenchmarkTransactionWindow::new(start, end)); + timeout(Duration::from_secs(1), async { + loop { + if BenchmarkGeneratorState::from_u8( + metrics.benchmark_generator_state.load(Ordering::Acquire), + ) == BenchmarkGeneratorState::Finished + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("backpressured generator did not finish at the common cutoff"); + + assert_eq!(metrics.submitted_transactions.get(), 1); + assert_eq!(receiver.recv().await.unwrap().len(), 1); + assert!(receiver.try_recv().is_err()); + } } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 43e202fd..1fe59af4 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -11,7 +11,7 @@ use std::{ use ::prometheus::Registry; use eyre::{Context, Result, eyre}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, watch}; use crate::{ block_handler::{RealBlockHandler, RealCommitHandler}, @@ -19,7 +19,7 @@ use crate::{ config::{NodePrivateConfig, NodePublicConfig, Parameters}, core::Core, dag_state::{DagState, ProtocolConfig}, - metrics::{MetricReporter, Metrics}, + metrics::{BenchmarkTransactionWindow, MetricReporter, Metrics}, net_sync::NetworkSyncer, network::Network, prometheus, @@ -36,6 +36,18 @@ pub struct Validator { reporter: Arc, } +#[derive(Clone, Debug, Default)] +pub struct ValidatorStartOptions { + /// Hold the autonomous RBC-DAG carrier clock until the owner explicitly + /// activates it. This is a local-benchmark coordination primitive, not a + /// serialized validator configuration or a production liveness gate. + pub rbc_dag_clock_start_paused: bool, + /// Optional local-benchmark release latch. Finite transaction generators + /// begin their common warmup only after every coordinated protocol clock + /// has crossed the event-bridge/Core activation barrier. + pub transaction_generator_start: Option>>, +} + impl Validator { pub async fn start( authority: AuthorityIndex, @@ -45,6 +57,30 @@ impl Validator { parameters: Parameters, byzantine_strategy: String, consensus: String, + ) -> Result { + Self::start_with_options( + authority, + committee, + public_config, + private_config, + parameters, + byzantine_strategy, + consensus, + ValidatorStartOptions::default(), + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn start_with_options( + authority: AuthorityIndex, + committee: Arc, + public_config: NodePublicConfig, + private_config: NodePrivateConfig, + parameters: Parameters, + byzantine_strategy: String, + consensus: String, + start_options: ValidatorStartOptions, ) -> Result { let protocol_config = ProtocolConfig::from_selection( &consensus, @@ -59,6 +95,16 @@ impl Validator { "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" )); } + if start_options.rbc_dag_clock_start_paused + && (!public_config.parameters.starfish_rbc_dag_autonomous_clock + || !public_config + .parameters + .starfish_rbc_dag_embedded_rbc_authority) + { + return Err(eyre!( + "A coordinated RBC-DAG clock start requires autonomous embedded authority" + )); + } if public_config .parameters .starfish_rbc_dag_embedded_rbc_authority @@ -207,12 +253,13 @@ impl Validator { // Rest of the function remains the same let (block_handler, block_sender) = RealBlockHandler::new(&committee); - TransactionGenerator::start( + TransactionGenerator::start_with_gate( block_sender, authority, parameters, public_config.clone(), metrics.clone(), + start_options.transaction_generator_start.clone(), ); let commit_handler = @@ -282,6 +329,7 @@ impl Validator { partial_sig_rx, bls_cert_aggregator, bls_signer_for_service, + start_options.rbc_dag_clock_start_paused, ) .await; @@ -303,6 +351,17 @@ impl Validator { self.reporter.clone() } + pub fn is_finished(&self) -> bool { + self.network_broadcaster.is_finished() || self.metrics_handle.is_finished() + } + + pub async fn activate_starfish_rbc_dag_clock(&self) -> Result<()> { + self.network_broadcaster + .activate_starfish_rbc_dag_clock() + .await + .map_err(|error| eyre!(error)) + } + pub async fn await_completion( self, ) -> ( @@ -316,8 +375,15 @@ impl Validator { } pub async fn stop(self) { - self.network_broadcaster.shutdown().await; - self.metrics_handle.abort(); + let Self { + network_broadcaster, + metrics_handle, + metrics: _, + reporter: _, + } = self; + metrics_handle.abort(); + let _ = metrics_handle.await; + network_broadcaster.shutdown().await; // Give time for background Worker tasks to detect channel closures and exit, // and for TCP sockets to fully release. tokio::time::sleep(std::time::Duration::from_secs(2)).await; @@ -714,29 +780,40 @@ mod smoke_tests { "autonomous mode must not claim direct-round comparison" ); if embedded_rbc_authority { + for message_kind in [ + "rbc_initial", + "rbc_echo", + "rbc_ready", + "rbc_header_request", + "rbc_header_response", + "batch", + "missing_parents", + "missing_tx_data", + ] { + assert_eq!( + metrics + .network_message_bytes_sent_total + .with_label_values(&[message_kind]) + .get(), + 0, + "standalone RBC-DAG mode must not send legacy {message_kind} traffic" + ); + assert_eq!( + metrics + .network_message_bytes_received_total + .with_label_values(&[message_kind]) + .get(), + 0, + "standalone RBC-DAG mode must not receive legacy {message_kind} traffic" + ); + } assert!( metrics .network_message_bytes_sent_total - .with_label_values(&["rbc_initial"]) + .with_label_values(&["rbc_dag_shadow_carrier"]) .get() > 0, - "direct INIT remains the application/payload transport" - ); - assert_eq!( - metrics - .network_message_bytes_sent_total - .with_label_values(&["rbc_echo"]) - .get(), - 0, - "direct RBC ECHO must be disabled under embedded authority" - ); - assert_eq!( - metrics - .network_message_bytes_sent_total - .with_label_values(&["rbc_ready"]) - .get(), - 0, - "direct RBC READY must be disabled under embedded authority" + "standalone RBC-DAG carrier transport must be active" ); } } diff --git a/crates/starfish/Cargo.toml b/crates/starfish/Cargo.toml index 27b05bc6..947aea33 100644 --- a/crates/starfish/Cargo.toml +++ b/crates/starfish/Cargo.toml @@ -10,6 +10,7 @@ edition = "2021" clap = { workspace = true } color-eyre = { workspace = true } eyre = { workspace = true } +futures = { workspace = true } prettytable-rs = "0.10" starfish-core = { path = "../starfish-core" } tokio = { workspace = true } diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 15d5f838..5bff7ddd 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -3,9 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::{ + collections::BTreeSet, fs, - net::{IpAddr, Ipv4Addr}, - path::PathBuf, + net::{IpAddr, Ipv4Addr, TcpListener, TcpStream}, + path::{Path, PathBuf}, sync::{ Arc, atomic::{AtomicBool, AtomicU64, Ordering}, @@ -16,21 +17,29 @@ use std::{ use clap::Parser; use eyre::{Context, Result}; +use futures::future::try_join_all; use prettytable::format; use starfish_core::{ - ByzantineStrategy, committee::Committee, config::{ DisseminationMode, ImportExport, NodeParameters, NodePrivateConfig, NodePublicConfig, Parameters, StorageBackend, TransactionMode, }, - metrics::Metrics, + metrics::{ + AutonomousClockBenchmarkSnapshot, BenchmarkGeneratorState, BenchmarkTransactionWindow, + LocalBenchmarkTransactionOutcome, Metrics, + }, types::AuthorityIndex, - validator::Validator, + validator::{Validator, ValidatorStartOptions}, }; -use tokio::time::Instant; +use tokio::{sync::watch, time::Instant}; use tracing_subscriber::{EnvFilter, filter::LevelFilter, fmt}; +// Network workers bind active sockets at `listener_port * 10`. Keep those +// derived ports below the conventional Linux ephemeral range so repeated +// local experiments cannot collide with an unrelated outbound connection. +const LOCAL_BENCHMARK_MAX_ACTIVE_BIND_PORT: u16 = 32_767; + #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Args { @@ -202,6 +211,11 @@ enum Operation { starfish_rbc_dag_shadow_buffered_wal: bool, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, + /// Add an offset to the local benchmark's fixed network and metrics + /// ports. This is useful for isolated repeated experiments and avoids + /// silently sharing `SO_REUSEPORT` listeners with a stale run. + #[clap(long, value_name = "INT", default_value_t = 0)] + port_offset: u16, /// Dissemination mode override: /// protocol-default | pull | push-causal | push-useful #[clap(long, value_name = "STRING")] @@ -315,6 +329,7 @@ async fn main() -> Result<()> { starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_dag_shadow_buffered_wal, duration_secs, + port_offset, dissemination_mode, } => { let mut node_parameters = NodeParameters::default_with_latency(mimic_extra_latency); @@ -344,6 +359,7 @@ async fn main() -> Result<()> { node_parameters, consensus_protocol, duration_secs, + port_offset, ) .await?; } @@ -411,14 +427,116 @@ fn benchmark_genesis( Ok(()) } +fn local_benchmark_private_configs( + base_dir: &Path, + committee_size: usize, +) -> Vec { + NodePrivateConfig::new_for_benchmarks(base_dir, committee_size) + .into_iter() + .enumerate() + .map(|(authority, mut private_config)| { + private_config.storage_path = base_dir.join(format!("node-{authority}")).join( + NodePrivateConfig::default_storage_path(authority as AuthorityIndex), + ); + private_config + }) + .collect() +} + +async fn stop_local_benchmark_validators(validators: Vec) { + let mut stops = tokio::task::JoinSet::new(); + for validator in validators { + stops.spawn(validator.stop()); + } + while let Some(result) = stops.join_next().await { + if let Err(error) = result { + tracing::warn!("Validator shutdown task failed: {error}"); + } + } +} + +fn local_benchmark_topology_ready( + metrics: &[Arc], + expected_peer_subscriptions: i64, +) -> Result { + local_benchmark_topology_state( + metrics.iter().map(|metrics| { + ( + metrics.metrics_active.load(Ordering::Relaxed), + metrics.subscribed_to_peers.get(), + metrics.subscribed_by_peers.get(), + ) + }), + expected_peer_subscriptions, + ) +} + +fn local_benchmark_topology_state( + states: impl IntoIterator, + expected_peer_subscriptions: i64, +) -> Result { + let states = states.into_iter().collect::>(); + eyre::ensure!( + !states.iter().any(|(active, _, _)| *active), + "A transaction generator opened before the complete peer subscription mesh" + ); + Ok(states.iter().all(|(_, subscribed_to, subscribed_by)| { + *subscribed_to == expected_peer_subscriptions + && *subscribed_by == expected_peer_subscriptions + })) +} + +/// Assign the configured aggregate transaction load exclusively to honest +/// validators. Byzantine validators exercise protocol behavior only: making +/// their deliberately withheld/dropped transactions part of the mandatory +/// drain target would conflate adversarial dissemination with lost honest +/// work. The remainder is distributed deterministically by authority order so +/// the per-validator rates sum to the exact requested aggregate load. +fn local_benchmark_generator_loads( + committee_size: usize, + aggregate_load: usize, + num_byzantine_nodes: usize, +) -> Result<(BTreeSet, Vec)> { + let byzantine_authorities = (0..committee_size) + .filter(|authority| *authority % 3 == 0 && *authority / 3 < num_byzantine_nodes) + .collect::>(); + eyre::ensure!( + byzantine_authorities.len() == num_byzantine_nodes, + "requested {num_byzantine_nodes} Byzantine validators, but the local benchmark layout can place only {} in a committee of {committee_size}", + byzantine_authorities.len(), + ); + let honest_count = committee_size.saturating_sub(byzantine_authorities.len()); + eyre::ensure!( + honest_count > 0, + "a local benchmark requires at least one honest transaction generator" + ); + let base = aggregate_load / honest_count; + let remainder = aggregate_load % honest_count; + let mut honest_rank = 0usize; + let loads = (0..committee_size) + .map(|authority| { + if byzantine_authorities.contains(&authority) { + 0 + } else { + let load = base + usize::from(honest_rank < remainder); + honest_rank += 1; + load + } + }) + .collect::>(); + debug_assert_eq!(loads.iter().sum::(), aggregate_load); + Ok((byzantine_authorities, loads)) +} + async fn local_benchmark( committee_size: usize, - mut load: usize, + load: usize, num_byzantine_nodes: usize, byzantine_strategy: String, node_parameters: NodeParameters, consensus_protocol: String, duration_secs: u64, + port_offset: u16, ) -> Result<()> { eyre::ensure!( duration_secs > 0, @@ -480,34 +598,52 @@ async fn local_benchmark( ); } println!("Duration: {duration_secs} seconds"); + println!("Local port offset: {port_offset}"); println!("===========================\n"); let ips = vec![IpAddr::V4(Ipv4Addr::LOCALHOST); committee_size]; let committee = Committee::new_for_benchmarks(committee_size); - load /= committee.len(); - let mut parameters = Parameters::almost_default(load); - parameters.benchmark_duration = Some(Duration::from_secs(duration_secs)); - // Equivocating Byzantine strategies must not generate transactions. - let mut byzantine_parameters = parameters.clone(); - if ByzantineStrategy::from_strategy_str(&byzantine_strategy) - .is_some_and(|s| s.is_equivocating()) - { - byzantine_parameters.load = 0; + let (byzantine_authorities, generator_loads) = + local_benchmark_generator_loads(committee_size, load, num_byzantine_nodes)?; + if !byzantine_authorities.is_empty() { + println!( + "Byzantine transaction generators: disabled; aggregate load redistributed across {} honest validators", + committee_size.saturating_sub(byzantine_authorities.len()), + ); } + let mut parameters = Parameters::almost_default(0); + parameters.benchmark_duration = Some(Duration::from_secs(duration_secs)); let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); + validate_local_benchmark_port_offset(&public_config, port_offset)?; + let public_config = public_config.with_port_offset(port_offset); + preflight_local_benchmark_ports(&public_config)?; let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; let starfish_rbc_dag_autonomous_clock_expected = node_parameters.starfish_rbc_dag_autonomous_clock; let starfish_rbc_dag_embedded_rbc_authority_expected = node_parameters.starfish_rbc_dag_embedded_rbc_authority; + let coordinated_rbc_dag_clock_start = starfish_rbc_dag_autonomous_clock_expected + && starfish_rbc_dag_embedded_rbc_authority_expected; // Create temporary directories for each validator - let base_dir = PathBuf::from("local-benchmark"); + // Isolate storage by the same explicit run namespace as the sockets. + // A stale or intentionally concurrent benchmark on another offset must + // not keep RocksDB/WAL handles open underneath this run's cleanup. + let base_dir = PathBuf::from(format!("local-benchmark-{port_offset}")); fs::create_dir_all(&base_dir)?; + // Generate the benchmark key material before any validator starts. Doing + // this inside the startup loop regenerates the entire committee keyset for + // every authority; once the first quorum is live, that CPU work lets its + // autonomous clock run far ahead of the validators still being prepared. + let private_configs = local_benchmark_private_configs(&base_dir, committee_size); - let mut handles = Vec::with_capacity(committee_size); - let mut abort_handles = Vec::with_capacity(committee_size); + let mut validators = Vec::with_capacity(committee_size); + let mut metrics_of_all_validators = Vec::with_capacity(committee_size); let mut metrics_of_honest_validators = Vec::new(); let mut reporters_of_honest_validators = Vec::new(); + // Every local benchmark, not only RBC-DAG, uses one absolute offered-load + // window. Production Validator::start remains ungated. + let (transaction_generator_start_tx, transaction_generator_start_rx) = + watch::channel(None::); // Create a flag to signal when the benchmark is complete let running = Arc::new(AtomicBool::new(true)); @@ -518,7 +654,7 @@ async fn local_benchmark( run_with_progress(running.clone(), elapsed_seconds.clone()); // Start all validators - for authority in 0..committee_size { + for (authority, private_config) in private_configs.into_iter().enumerate() { tracing::warn!( "Starting node {authority} in local \ benchmark mode (committee size: {committee_size})" @@ -535,9 +671,6 @@ async fn local_benchmark( )); } } - let mut private_configs = - NodePrivateConfig::new_for_benchmarks(&working_dir, committee_size); - let private_config = private_configs.remove(authority); match fs::create_dir_all(&private_config.storage_path) { Ok(_) => {} Err(e) => { @@ -547,53 +680,49 @@ async fn local_benchmark( )); } } - let is_byzantine = authority.is_multiple_of(3) && authority / 3 < num_byzantine_nodes; + let is_byzantine = byzantine_authorities.contains(&authority); + let mut generator_parameters = parameters.clone(); + generator_parameters.load = generator_loads[authority]; + let start_options = ValidatorStartOptions { + rbc_dag_clock_start_paused: coordinated_rbc_dag_clock_start, + transaction_generator_start: Some(transaction_generator_start_rx.clone()), + }; let validator = if is_byzantine { - Validator::start( + Validator::start_with_options( authority as AuthorityIndex, committee.clone(), public_config.clone(), private_config, - byzantine_parameters.clone(), + generator_parameters, byzantine_strategy.clone(), consensus_protocol.clone(), + start_options, ) .await? } else { - Validator::start( + Validator::start_with_options( authority as AuthorityIndex, committee.clone(), public_config.clone(), private_config, - parameters.clone(), + generator_parameters, "honest".to_string(), consensus_protocol.clone(), + start_options, ) .await? }; let validator_metrics = validator.metrics(); + metrics_of_all_validators.push(Arc::clone(&validator_metrics)); if !is_byzantine { metrics_of_honest_validators.push(Arc::clone(&validator_metrics)); reporters_of_honest_validators.push(validator.reporter()) } - // Use the same pattern as the run method - let handle = tokio::spawn(async move { - let (network_result, _metrics_result) = validator.await_completion().await; - if starfish_rbc_dag_autonomous_clock_expected { - validator_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - } else if starfish_rbc_dag_shadow_expected { - validator_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - network_result - }); - abort_handles.push(handle.abort_handle()); - handles.push(handle); + validators.push(validator); } - if starfish_rbc_dag_shadow_expected { + if starfish_rbc_dag_shadow_expected && !coordinated_rbc_dag_clock_start { let ready = tokio::time::timeout(Duration::from_secs(30), async { loop { if metrics_of_honest_validators.iter().all(|metrics| { @@ -610,9 +739,8 @@ async fn local_benchmark( }) .await; if ready.is_err() { - for abort_handle in &abort_handles { - abort_handle.abort(); - } + running.store(false, Ordering::SeqCst); + stop_local_benchmark_validators(validators).await; fs::remove_dir_all(&base_dir)?; let mode = if starfish_rbc_dag_autonomous_clock_expected { "autonomous clock" @@ -625,26 +753,110 @@ async fn local_benchmark( } } - // `duration_secs` is an active transaction-submission window, not a - // process-lifetime cutoff. Every finite generator holds metrics inactive - // through connection warmup, then opens this latch immediately before its - // first batch. Start the benchmark only after every honest validator has - // crossed that boundary. - tokio::time::timeout(Duration::from_secs(30), async { + // A ready protocol actor is not yet a ready benchmark network. Require + // every honest validator to observe the complete logical subscription + // mesh before recording baselines. This fails closed if an earlier core + // failure leaves independent TCP workers alive or if startup is only + // partially connected. + let expected_peer_subscriptions = + i64::try_from(committee_size.saturating_sub(1)).unwrap_or(i64::MAX); + let topology_ready = tokio::time::timeout(Duration::from_secs(30), async { loop { - if metrics_of_honest_validators - .iter() - .all(|metrics| metrics.metrics_active.load(Ordering::Relaxed)) - { + if local_benchmark_topology_ready( + &metrics_of_all_validators, + expected_peer_subscriptions, + )? { break; } tokio::time::sleep(Duration::from_millis(25)).await; } + Ok::<(), eyre::Report>(()) }) - .await - .wrap_err("transaction generators did not open the active benchmark window")?; - println!("Active transaction window started ({duration_secs} seconds)"); + .await; + if !matches!(topology_ready, Ok(Ok(()))) { + running.store(false, Ordering::SeqCst); + stop_local_benchmark_validators(validators).await; + fs::remove_dir_all(&base_dir)?; + eyre::bail!( + "Local benchmark did not establish the complete {expected_peer_subscriptions}-peer subscription mesh on every honest validator" + ); + } + + if coordinated_rbc_dag_clock_start { + let activated = tokio::time::timeout( + Duration::from_secs(30), + try_join_all( + validators + .iter() + .map(Validator::activate_starfish_rbc_dag_clock), + ), + ) + .await; + if let Ok(Err(error)) = &activated { + tracing::error!("Failed to activate a coordinated RBC-DAG clock: {error}"); + } + if !matches!(activated, Ok(Ok(_))) { + running.store(false, Ordering::SeqCst); + stop_local_benchmark_validators(validators).await; + fs::remove_dir_all(&base_dir)?; + eyre::bail!( + "Local benchmark could not activate every RBC-DAG clock after full topology readiness" + ); + } + + // The service activation acknowledgment only publishes the ordered + // ClockActivated event. Wait until each event bridge has processed it, + // released the authoritative Core producer, and marked the clock live. + let clock_ready = tokio::time::timeout(Duration::from_secs(30), async { + loop { + local_benchmark_topology_ready( + &metrics_of_all_validators, + expected_peer_subscriptions, + )?; + if metrics_of_all_validators + .iter() + .all(|metrics| metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1) + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Ok::<(), eyre::Report>(()) + }) + .await; + if !matches!(clock_ready, Ok(Ok(()))) { + running.store(false, Ordering::SeqCst); + stop_local_benchmark_validators(validators).await; + fs::remove_dir_all(&base_dir)?; + eyre::bail!("RBC-DAG clocks did not activate before the transaction window opened"); + } + } + + let generators_waiting = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if metrics_of_all_validators.iter().all(|metrics| { + BenchmarkGeneratorState::from_u8( + metrics.benchmark_generator_state.load(Ordering::Acquire), + ) == BenchmarkGeneratorState::Waiting + }) { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + if generators_waiting.is_err() { + running.store(false, Ordering::SeqCst); + stop_local_benchmark_validators(validators).await; + fs::remove_dir_all(&base_dir)?; + eyre::bail!("transaction generators did not reach the coordinated waiting state"); + } + // Clear warmup samples and capture every baseline before publishing the + // release. No generator can submit before the common absolute start. + for reporter in &reporters_of_honest_validators { + reporter.reset_for_benchmark_window(); + } let autonomous_clock_baselines = starfish_rbc_dag_autonomous_clock_expected.then(|| { metrics_of_honest_validators .iter() @@ -655,59 +867,263 @@ async fn local_benchmark( .iter() .map(|metrics| metrics.local_benchmark_counter_baseline()) .collect::>(); + let sequenced_baselines = metrics_of_honest_validators + .iter() + .map(|metrics| metrics.sequenced_transactions_total.get()) + .collect::>(); + let cutoff_sequenced_baselines = metrics_of_honest_validators + .iter() + .map(|metrics| metrics.sequenced_transactions_cutoff_total.get()) + .collect::>(); + let honest_submitted_baselines = metrics_of_honest_validators + .iter() + .map(|metrics| metrics.submitted_transactions.get()) + .collect::>(); + let window_start = Instant::now() + Duration::from_millis(100); + let window_end = window_start + .checked_add(Duration::from_secs(duration_secs)) + .ok_or_else(|| eyre::eyre!("benchmark duration exceeds the monotonic clock range"))?; + let transaction_window = BenchmarkTransactionWindow::new(window_start, window_end) + .expect("positive local benchmark duration must form a valid window"); + for metrics in &metrics_of_all_validators { + let cutoff_micros = window_end + .saturating_duration_since(metrics.validator_start) + .as_micros() + .min(u64::MAX as u128) as u64; + metrics + .benchmark_transaction_cutoff_micros + .store(cutoff_micros, Ordering::Release); + } + transaction_generator_start_tx.send_replace(Some(transaction_window)); + tokio::time::sleep_until(window_start).await; - // Run for specified duration - tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(duration_secs)) => { - // Signal the progress display to stop - running.store(false, Ordering::SeqCst); - println!(); - println!("Benchmark completed after {duration_secs} seconds"); - // Display metrics - Metrics::aggregate_and_display( - metrics_of_honest_validators, - reporters_of_honest_validators, - duration_secs, - committee_size, - starfish_rbc_dag_shadow_expected, - starfish_rbc_dag_autonomous_clock_expected, - starfish_rbc_dag_embedded_rbc_authority_expected, - autonomous_clock_baselines.clone(), - Some(counter_baselines.clone()), + let active_window_ready = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let states = metrics_of_all_validators + .iter() + .map(|metrics| { + BenchmarkGeneratorState::from_u8( + metrics.benchmark_generator_state.load(Ordering::Acquire), + ) + }) + .collect::>(); + eyre::ensure!( + !states.contains(&BenchmarkGeneratorState::Failed), + "a transaction generator failed while opening the common window" ); - - // Abort all tasks - for abort_handle in abort_handles { - abort_handle.abort(); + if states + .iter() + .all(|state| *state == BenchmarkGeneratorState::Active) + { + break; } - - // Clean up - fs::remove_dir_all(base_dir)?; - Ok(()) + tokio::task::yield_now().await; } + Ok::<(), eyre::Report>(()) + }) + .await; + if !matches!(active_window_ready, Ok(Ok(()))) { + running.store(false, Ordering::SeqCst); + stop_local_benchmark_validators(validators).await; + fs::remove_dir_all(&base_dir)?; + eyre::bail!("transaction generators did not open the common active benchmark window"); + } + println!("Active transaction window started ({duration_secs} seconds)"); + + // Run for the requested active duration, but fail if any validator exits. + // Keep ownership of the validators so the normal path can use their + // graceful shutdown rather than aborting wrapper tasks and leaking socket + // workers into subsequent latency experiments. + let completed_full_duration = tokio::select! { + _ = tokio::time::sleep_until(window_end) => true, _ = async { - for handle in handles { - if let Err(e) = handle.await { - tracing::warn!("Validator terminated with error: {}", e); + loop { + if validators.iter().any(Validator::is_finished) { + break; } + tokio::time::sleep(Duration::from_millis(25)).await; } - } => { - println!("All validators completed before timeout"); - Metrics::aggregate_and_display( - metrics_of_honest_validators, - reporters_of_honest_validators, - duration_secs, - committee_size, - starfish_rbc_dag_shadow_expected, - starfish_rbc_dag_autonomous_clock_expected, - starfish_rbc_dag_embedded_rbc_authority_expected, - autonomous_clock_baselines, - Some(counter_baselines), - ); - fs::remove_dir_all(base_dir)?; - eyre::bail!("All validators completed before the requested benchmark duration") - } + } => false, + }; + + // Close protocol/window metrics at the exact common cutoff even if a + // delayed generator task still has a scheduled pre-end batch to publish. + for metrics in &metrics_of_all_validators { + metrics.metrics_active.store(false, Ordering::Release); } + let autonomous_clock_cutoffs: Option> = + starfish_rbc_dag_autonomous_clock_expected.then(|| { + metrics_of_honest_validators + .iter() + .map(|metrics| metrics.autonomous_clock_benchmark_snapshot()) + .collect() + }); + let counter_cutoffs = metrics_of_honest_validators + .iter() + .map(|metrics| metrics.local_benchmark_counter_baseline()) + .collect::>(); + let cutoff_committed_transactions = metrics_of_honest_validators + .iter() + .zip(&cutoff_sequenced_baselines) + .map(|(metrics, baseline)| { + metrics + .sequenced_transactions_cutoff_total + .get() + .saturating_sub(*baseline) + }) + .sum::() + / metrics_of_honest_validators.len() as u64; + + let drain_started = Instant::now(); + let generators_finished = completed_full_duration + && matches!( + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let states = metrics_of_all_validators + .iter() + .map(|metrics| { + BenchmarkGeneratorState::from_u8( + metrics.benchmark_generator_state.load(Ordering::Acquire), + ) + }) + .collect::>(); + eyre::ensure!( + !states.contains(&BenchmarkGeneratorState::Failed), + "a transaction generator failed during the active window" + ); + if states + .iter() + .all(|state| *state == BenchmarkGeneratorState::Finished) + { + break; + } + tokio::task::yield_now().await; + } + Ok::<(), eyre::Report>(()) + }) + .await, + Ok(Ok(())) + ); + + // Byzantine generators have zero configured load, and offered work is + // defined explicitly from honest generators so adversarial strategies can + // neither inflate nor make the mandatory drain target unattainable. + let offered_transactions = metrics_of_honest_validators + .iter() + .zip(&honest_submitted_baselines) + .map(|(metrics, baseline)| { + metrics + .submitted_transactions + .get() + .saturating_sub(*baseline) + }) + .sum::(); + let pipeline_observed_through_target = |metrics: &Metrics, sequenced_baseline: u64| { + !starfish_rbc_dag_embedded_rbc_authority_expected + || offered_transactions == 0 + || metrics.starfish_rbc_dag_frontier_applied_sequenced_transactions() + >= sequenced_baseline.saturating_add(offered_transactions) + }; + let drain_complete = generators_finished + && tokio::time::timeout(Duration::from_secs(30), async { + loop { + if metrics_of_honest_validators + .iter() + .zip(&sequenced_baselines) + .all(|(metrics, baseline)| { + let sequenced = metrics + .sequenced_transactions_total + .get() + .saturating_sub(*baseline); + sequenced >= offered_transactions + && pipeline_observed_through_target(metrics, *baseline) + }) + { + break; + } + if validators.iter().any(Validator::is_finished) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .is_ok() + && metrics_of_honest_validators + .iter() + .zip(&sequenced_baselines) + .all(|(metrics, baseline)| { + metrics + .sequenced_transactions_total + .get() + .saturating_sub(*baseline) + == offered_transactions + }) + && metrics_of_honest_validators + .iter() + .zip(&sequenced_baselines) + .all(|(metrics, baseline)| pipeline_observed_through_target(metrics, *baseline)); + let drain_elapsed = drain_started.elapsed(); + for metrics in &metrics_of_all_validators { + metrics + .transaction_metrics_active + .store(false, Ordering::Release); + } + let eventual_committed_transactions = metrics_of_honest_validators + .iter() + .zip(&sequenced_baselines) + .map(|(metrics, baseline)| { + metrics + .sequenced_transactions_total + .get() + .saturating_sub(*baseline) + }) + .sum::() + / metrics_of_honest_validators.len() as u64; + let transaction_outcome = LocalBenchmarkTransactionOutcome { + offered_transactions, + cutoff_committed_transactions, + eventual_committed_transactions, + drain_elapsed, + drain_complete, + }; + + running.store(false, Ordering::SeqCst); + println!(); + if completed_full_duration { + println!("Benchmark completed after {duration_secs} seconds"); + } else { + println!("A validator completed before timeout"); + } + Metrics::aggregate_and_display( + metrics_of_honest_validators, + reporters_of_honest_validators, + duration_secs, + committee_size, + starfish_rbc_dag_shadow_expected, + starfish_rbc_dag_autonomous_clock_expected, + starfish_rbc_dag_embedded_rbc_authority_expected, + autonomous_clock_baselines, + autonomous_clock_cutoffs, + Some(counter_baselines), + Some(counter_cutoffs), + Some(transaction_outcome), + ); + stop_local_benchmark_validators(validators).await; + fs::remove_dir_all(base_dir)?; + eyre::ensure!( + completed_full_duration, + "A validator completed before the requested benchmark duration" + ); + eyre::ensure!( + generators_finished, + "A transaction generator failed to finish the common active window" + ); + eyre::ensure!( + drain_complete, + "Active-window transaction drain was incomplete: offered={offered_transactions}, eventual average committed={eventual_committed_transactions}" + ); + Ok(()) } /// Boot a single validator node. @@ -892,6 +1308,58 @@ fn ensure_starfish_rbc_protocol_instance( } } +fn validate_local_benchmark_port_offset( + public_config: &NodePublicConfig, + port_offset: u16, +) -> Result<()> { + eyre::ensure!( + public_config.all_network_addresses().all(|address| { + address + .port() + .checked_add(port_offset) + .is_some_and(|port| port <= LOCAL_BENCHMARK_MAX_ACTIVE_BIND_PORT / 10) + }), + "local benchmark port offset {port_offset} places a derived active-bind port in the OS ephemeral range; choose a smaller offset" + ); + Ok(()) +} + +fn preflight_local_benchmark_ports(public_config: &NodePublicConfig) -> Result<()> { + let mut addresses = BTreeSet::new(); + for address in public_config.all_network_addresses() { + addresses.insert(address); + let mut active = address; + active.set_port( + address + .port() + .checked_mul(10) + .expect("validated local benchmark active port"), + ); + addresses.insert(active); + } + addresses.extend(public_config.all_metric_addresses()); + + // The network listeners enable SO_REUSEPORT. A plain bind probe can + // therefore succeed even while a stale benchmark is still serving the + // same address. Probe for an existing listener first, then reserve every + // address for the duration of this check. + for address in &addresses { + eyre::ensure!( + TcpStream::connect_timeout(address, Duration::from_millis(25)).is_err(), + "local benchmark port {address} is already served by another process" + ); + } + + let mut reservations = Vec::with_capacity(addresses.len()); + for address in addresses { + reservations.push( + TcpListener::bind(address) + .wrap_err_with(|| format!("local benchmark port {address} is already in use"))?, + ); + } + Ok(()) +} + fn ipv4_add_offset(base: Ipv4Addr, offset: usize) -> Result { let offset = u32::try_from(offset).context("validator count exceeds IPv4 offset range")?; let next = u32::from(base) @@ -949,12 +1417,23 @@ pub fn default_table_format() -> format::TableFormat { #[cfg(test)] mod tests { - use std::net::Ipv4Addr; + use std::{ + net::{IpAddr, Ipv4Addr, TcpListener}, + path::PathBuf, + }; use clap::Parser; - use super::{Args, Operation, ensure_starfish_rbc_protocol_instance, ipv4_add_offset}; - use starfish_core::config::NodeParameters; + use super::{ + Args, Operation, ensure_starfish_rbc_protocol_instance, ipv4_add_offset, + local_benchmark_generator_loads, local_benchmark_private_configs, + local_benchmark_topology_state, preflight_local_benchmark_ports, + validate_local_benchmark_port_offset, + }; + use starfish_core::{ + config::{NodeParameters, NodePrivateConfig, NodePublicConfig}, + types::AuthorityIndex, + }; #[test] fn ipv4_add_offset_crosses_octet_boundary() { @@ -971,6 +1450,22 @@ mod tests { assert!(ipv4_add_offset(base, 1).is_err()); } + #[test] + fn local_benchmark_disables_byzantine_generators_and_preserves_aggregate_load() { + let (byzantine, loads) = local_benchmark_generator_loads(10, 1_003, 2).unwrap(); + + assert_eq!(byzantine.into_iter().collect::>(), vec![0, 3]); + assert_eq!(loads[0], 0); + assert_eq!(loads[3], 0); + assert_eq!(loads.iter().sum::(), 1_003); + assert_eq!(loads, vec![0, 126, 126, 0, 126, 125, 125, 125, 125, 125]); + } + + #[test] + fn local_benchmark_rejects_unplaceable_byzantine_count() { + assert!(local_benchmark_generator_loads(4, 100, 3).is_err()); + } + #[test] fn dry_run_parses_block_authentication_separately_from_consensus() { let args = Args::try_parse_from([ @@ -1014,6 +1509,8 @@ mod tests { "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-dag-shadow-buffered-wal", + "--port-offset", + "2500", ]) .unwrap(); @@ -1024,6 +1521,7 @@ mod tests { starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_dag_shadow_buffered_wal, + port_offset, .. } = args.operation else { @@ -1035,6 +1533,7 @@ mod tests { assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_dag_embedded_rbc_authority); assert!(starfish_rbc_dag_shadow_buffered_wal); + assert_eq!(port_offset, 2500); } #[test] @@ -1055,4 +1554,56 @@ mod tests { assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); } + + #[test] + fn local_benchmark_port_offset_stays_below_ephemeral_active_ports() { + let safe = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST); 10], None); + validate_local_benchmark_port_offset(&safe, 200).unwrap(); + + let ephemeral = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST); 10], None); + assert!(validate_local_benchmark_port_offset(&ephemeral, 3_500).is_err()); + assert!(validate_local_benchmark_port_offset(&ephemeral, u16::MAX).is_err()); + } + + #[test] + fn local_benchmark_private_configs_preserve_per_authority_storage_layout() { + let base_dir = PathBuf::from("local-benchmark-config-test"); + let committee_size = 4; + let private_configs = local_benchmark_private_configs(&base_dir, committee_size); + + assert_eq!(private_configs.len(), committee_size); + for (authority, private_config) in private_configs.into_iter().enumerate() { + assert_eq!(private_config.mac_keys.len(), committee_size); + assert_eq!( + private_config.storage_path, + base_dir.join(format!("node-{authority}")).join( + NodePrivateConfig::default_storage_path(authority as AuthorityIndex) + ) + ); + } + } + + #[test] + fn local_benchmark_preflight_rejects_an_existing_listener() { + let public_config = + NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None) + .with_port_offset(1_400); + let address = public_config.network_address(0).unwrap(); + let _listener = match TcpListener::bind(address) { + Ok(listener) => listener, + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(error) => panic!("failed to bind preflight test listener: {error}"), + }; + + assert!(preflight_local_benchmark_ports(&public_config).is_err()); + } + + #[test] + fn local_benchmark_topology_readiness_fails_closed_before_baselining() { + assert!(!local_benchmark_topology_state([(false, 2, 3)], 3).unwrap()); + assert!(local_benchmark_topology_state([(false, 3, 3)], 3).unwrap()); + assert!(local_benchmark_topology_state([(true, 3, 3)], 3).is_err()); + } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 59a9ae77..31c78c69 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,27 +1,25 @@ # Starfish-RBC-DAG protocol design -Status: milestone-seven committed-frontier output prototype; end-to-end safety/liveness proof, -proof-safe retirement, and full validator recovery remain incomplete - -The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not -implemented. The staged prototype runs under `starfish-rbc`: direct-header comparison uses -`--starfish-rbc-dag-shadow`, the independent carrier clock adds -`--starfish-rbc-dag-autonomous-clock`, and milestone five makes embedded carrier ECHO/READY the -only application-header certification authority with -`--starfish-rbc-dag-embedded-rbc-authority`. Direct INIT still transports the application payload, -but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add -`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Autonomous -carriers now create durably locked logical consensus vertices and the clean projection produces -Starfish commit/skip decisions. In embedded-authority mode, committed projected anchors now release -deterministic exact carrier-frontier deltas and the legacy Starfish committer is disabled. The -eventual protocol is new, not a transport option or a version-two alias for `starfish-rbc`. - -The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative -baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through -RBC-delivered dependency-closed headers, and sends one initial MAC tag to each recipient. This -document instead specifies an optimistic carrier DAG that embeds the reliable-broadcast transcript, -uses a complete committee-sized MAC vector on every MAC-authenticated carrier, and separates fast -carrier pacing from certified consensus ordering. +Status: standalone MAC-vector RBC-DAG prototype with authoritative optimistic delivery and +committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoint transfer, and +full validator recovery remain incomplete + +The provisional CLI name for the eventual protocol is `starfish-rbc-dag`; that selector is not yet +implemented. The prototype runs under `starfish-rbc`: direct-header comparison uses +`--starfish-rbc-dag-shadow`, and standalone authority additionally enables +`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority`. In standalone +mode the direct Starfish-RBC service is not started. Direct INIT, direct phase messages, direct +header pull, generic block batches, legacy missing-parent pull, and legacy transaction-data pull +have no certification, consensus, ordering, or output authority. Canonical application headers are +inside carriers; optional application bytes use the carrier envelope or the dedicated RBC-DAG +payload request/response path. Committed projected anchors release cumulative exact +carrier-frontier deltas, and those deltas are the sole application-ordering/output authority. + +The implemented direct [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains a separate +historical baseline: it sends INIT/ECHO/READY as direct messages and advances Starfish only through +its direct delivery path. This document specifies the standalone optimistic carrier DAG, its +four-phase weighted ECHO/VOTE/ACK/READY broadcast, its complete committee-sized MAC-vector +sidecar, and the separation between fast carrier pacing and logical consensus ordering. The two designs may share canonicalization, cryptography, storage, and benchmark code, but they are not wire-compatible and do not have the same proof obligations. Nothing in this document changes @@ -33,9 +31,10 @@ The objective is to recover the pipelining of an uncertified Starfish DAG withou optimistically received Byzantine blocks to affect safety: - a fast physical carrier DAG advances on a quorum of locally authenticated carriers; -- every application carrier is also the value of a Bracha reliable-broadcast instance; -- later carriers batch ECHO and READY statements for earlier carriers; -- only RBC-delivered, data-available information enters the logical consensus projection; and +- every carrier is the exact value of a weighted reliable-broadcast slot; +- later carriers batch ECHO, VOTE, ACK, and READY statements for earlier carriers; +- only authoritatively delivered, data-available exact prefixes enter the logical consensus + projection; and - committed consensus frontiers eventually order every honest on-prefix application carrier. The initial research mode sends the complete ordered MAC vector with every carrier. The vector is @@ -43,50 +42,33 @@ an authentication sidecar and is not part of the carrier digest. Ed25519, ML-DSA remain selectable outer-authentication baselines; changing that selector does not change the embedded RBC or consensus rules. -This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish -commit rules already exist. Milestone two provides a canonical codec plus deterministic -carrier/RBC, certified-projection, decision, and crash-journal models. Milestone three adds an -opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired -direct/shadow delivery observations. Milestone four adds an independently authenticated autonomous -heartbeat namespace, sequential `Q`-admitted carrier clock, bounded future buffering, exact-slot -synchronization, and clock-state metrics. Milestone five adds a version-two application carrier -containing the exact canonical application header, durable application-origin reconciliation, -immediate application/phase scheduling, and an opt-in authority boundary that prevents direct -ECHO/READY from certifying a header. Idle control heartbeats use the same resolved leader timeout as -the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application carriers and their -encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. Milestone six -adds independently numbered consensus vertices, quorum strong parents, objective Vote/NoVote -choices, exact contiguous delivery frontiers, durable local consensus locks, and a live committer -that consumes only RBC-delivered, data-available projected vertices. Milestone seven persists and -reconstructs the corresponding anchor/frontier state, applies exact closed-prefix deltas, and makes -those deltas the sole application output authority. - -The current authoritative mode changes header certification and application ordering. Direct INIT -is still the application-payload transport and is not a certification vote. Direct ECHO/READY and -the legacy Starfish committer cannot certify or output applications in this mode; only the -carrier-DAG projection's committed frontier deltas can do so. - -Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode -requires an identical recovered direct-header history, control-only autonomous history reopens -without direct headers, and autonomous application origins must match recovered direct history. -Every mode serves byte-identical exact-slot responses. This is not a full validator crash-recovery -claim. The authoritative direct -`starfish-rbc` baseline does not yet durably record its remote-slot -ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it -has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest -recovered own header dirty. Full validator restart remains fail-stop until those direct-RBC locks -are persisted and replayed; replaying only the observational shadow WAL cannot repair or safely -substitute for them. - -That isolation is logical, not physical: shadow frames share the validator's existing TCP -connections, outbound queues, bandwidth, and CPU process with direct RBC, so enabling the shadow -can perturb authoritative timing even though no shadow result is consumed by consensus. Version -one has no feature handshake. Every validator in a shadow run must use a binary that understands -the append-only shadow wire variants and the flag must be deployed committee-wide; an older peer -will reject an unknown bincode variant and may close the shared connection. Mixed-version or -partially enabled runs are not valid comparisons. Mirror and autonomous carriers derive distinct -authentication protocol instances so they cannot cross-admit, but this fail-closed boundary is not -a substitute for capability negotiation. +The implemented composition includes the canonical codec, deterministic four-phase reducer, +durable proof-critical journal, autonomous carrier clock, exact carrier recovery, optimistic and +certification latches, consensus projection, and cumulative committed-frontier output. Idle control +heartbeats use the resolved Starfish leader timeout (600 ms for Push/Starfish-RBC by default), but +application carriers and encodable phase follow-ups are event-driven. The V4 authoritative mode +promotes the proved `O`-ECHO predicate to delivery authority; `Q` READY remains a distinct slower +certification fact rather than a prerequisite for fast projection. + +Standalone authority is an end-to-end boundary. The direct Starfish-RBC actor is absent, generic +block dissemination and legacy pull messages are rejected or ignored, and the legacy Starfish +committer is disabled. Only durably authoritatively delivered carrier content, the typed +verified-payload ingress, projected consensus decisions, and committed frontier deltas can affect +application output. + +Restart coverage is deliberately scoped. The actor WAL replays retained exact content, ordered +ingress, local ECHO/VOTE/ACK/READY locks, delivery/promise locks, local outbound bytes, consensus +choices, projection state, and committed frontiers. Exact-slot responses are byte-identical when +the requested locally authored outbound carrier remains retained. This is not yet a full validator +crash-recovery, checkpoint-transfer, proof-safe retirement, or arbitrary late-node catch-up claim. + +The carrier plane shares the validator's existing TCP connections, outbound queues, bandwidth, and +CPU process. There is no feature handshake, so every validator in a run must understand the +append-only carrier messages and enable a homogeneous configuration. Mirror and autonomous modes +derive distinct authentication protocol instances. The authoritative durable path additionally +uses the V4 autonomous WAL filename and `SRD5` raw-record magic, preventing older promise semantics +or record layouts from being silently replayed as the current protocol. These fail-closed +namespaces are not a substitute for deployment negotiation. Shadow shutdown is bounded so an observational WAL failure cannot indefinitely block validator shutdown. If that timeout fires, the blocking worker may still hold the shadow WAL's single-writer @@ -104,19 +86,17 @@ separately and makes no crash-safety claim. This removes the known persistence o without changing the protocol reducer. The runtime also uses a fixed unsolicited-retention window only as a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the composition and resource bounds are completed, `starfish-rbc-dag` remains an experimental -shadow/reference implementation rather than a proven signature-free Starfish variant. +reference implementation rather than a proven signature-free Starfish variant. -The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified -Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not -payload reconstruction or the runtime transition from delivered acknowledgments to that input. - -Transaction bytes remain outside header RBC. The existing Reed-Solomon dissemination, -acknowledgment, reconstruction, and transaction-commitment checks remain responsible for data -availability. +Application bytes remain outside canonical carrier identity. The optional proactive bytes and +dedicated payload responses are untrusted transport until the sole verifier checks them against the +canonical application header and transaction commitment. The actor receives `DataAvailable` only +after Core has materialized the concrete block and its existing availability predicate succeeds; +neither carrier delivery nor a raw payload response may bypass that application DA gate. ## 2. Model and notation -Version one assumes: +The current prototype assumes: - one static, ordered, stake-weighted committee for a run; - Byzantine stake strictly below one third of total stake; @@ -126,22 +106,36 @@ Version one assumes: - no committee reconfiguration; and - no state retirement until a safe recovery watermark is proved. -For total committee stake `W`, use the repository's integer thresholds: +For one target slot, let `W` be total committee stake and `a` the target author's stake. The +implementation derives all weighted thresholds with checked integer arithmetic: ```text -Q = floor(2W / 3) + 1 -V = floor(W / 3) + 1 +F = floor((W - 1) / 3) maximum Byzantine stake under faults < W/3 +V = F + 1 READY validity/amplification threshold +Q = W - F READY certification and carrier quorum threshold + +U = W - a stake outside the target author +b = F - a residual Byzantine non-author stake, when a <= F +M = floor(U / 2) + 1 strict non-author majority +C = floor((U + b) / 2) + 1 convergence threshold +O = M + b authoritative optimistic ECHO threshold ``` -For equal stake and `n = 3f + 1`, these are `Q = 2f + 1` and `V = f + 1`. +`M`, `C`, and `O` apply when `a <= F`, so the target author may be Byzantine. ECHO, VOTE, and ACK +stake exclude that author. If `a > F`, the global fault bound makes the target author honest; the +receiver-specific authenticator on exact content is then an immediate authoritative-delivery +predicate, and the locally fixed high-stake author seeds READY for the independent certificate +path. READY always counts the whole committee. For equal stake and `n = 3f + 1`, `Q = 2f + 1` and +`V = f + 1`. Two independent round numbers are used: - `carrier_round` belongs to the fast physical DAG and advances from authenticated admission; -- `consensus_round` belongs to the certified logical Starfish projection. +- `consensus_round` belongs to the authoritative logical Starfish projection. There is no fixed mapping between them. Carrier rounds may run ahead while a consensus vertex is -waiting for RBC delivery, data availability, a leader decision, or certified strong parents. +waiting for authoritative carrier delivery, application data availability, a leader decision, or +eligible strong parents. ## 3. One physical DAG, two logical projections @@ -153,24 +147,26 @@ The same stored objects have two disjoint interpretations: 1. **Optimistic carrier projection.** Authenticated carriers and their weak parent references pace carrier creation and transport RBC statements. This projection is allowed to differ temporarily between honest validators. -2. **Certified consensus projection.** Eligible consensus vertices, immutable strong parents, and - certified delivery frontiers drive Starfish voting, certification, skip/commit decisions, and - linearization. Honest validators eventually agree on this projection. +2. **Authoritative consensus projection.** Authoritatively delivered and data-available consensus + vertices, immutable strong parents, and exact delivery frontiers drive Starfish voting, + certification, skip/commit decisions, and linearization. The `O` fast-delivery rule may admit a + vertex before its independent `Q`-READY certificate; honest validators eventually agree on the + safe projection. Weak carrier edges never become strong/order edges, even if their targets later deliver. A node must not construct its own filtered consensus parent set from an optimistic carrier after the fact: that would make the authenticated content have different consensus meaning at different nodes. This separation prevents a Byzantine carrier from poisoning an honest carrier. A quorum-sized weak -parent set can contain up to `f` selectively disseminated or invented Byzantine references. Waiting -for all such references to become RBC-delivered would make the honest child permanently unusable. +parent set can contain selectively disseminated or invented Byzantine references. Waiting for all +such references to become authoritatively delivered would make the honest child permanently unusable. Weak edges are therefore permanently nonblocking and nonordering. Only the explicitly encoded -strong parents and certified frontier constrain consensus. +strong parents and the authoritative exact frontier constrain consensus. ## 4. Canonical objects -The milestone-two codec implements the following logical types. Field widths, enum codes, maximum -lengths, and golden bytes are frozen before runtime integration. +The canonical codec implements the following logical types. Field widths, enum codes, maximum +lengths, and golden bytes are part of the implemented runtime contract. ```rust struct CarrierHeaderV1 { @@ -182,6 +178,7 @@ struct CarrierHeaderV1 { weak_parents: Vec, transactions_commitment: TransactionsCommitment, + application_header: Option, data_acknowledgments: Vec, phase_batch: Vec, consensus_vertex: Option, @@ -191,6 +188,8 @@ struct CarrierHeaderV1 { enum RbcPhaseStatementV1 { Echo { target: BlockReference }, Ready { target: BlockReference }, + Vote { target: BlockReference }, + Ack { target: BlockReference }, } struct ConsensusVertexV1 { @@ -212,8 +211,10 @@ enum LeaderChoiceV1 { ``` The author of an embedded phase statement or consensus vertex is the author of its enclosing -carrier. An outer authenticator therefore authenticates the whole batch without a separate tag or -signature per statement. +carrier. An outer authenticator therefore authenticates the ordered batch without a separate tag +or signature per statement. `application_header = None` is a control-only V1 carrier; +`Some(exact_header)` selects the V2 identity/wire grammar and binds the complete canonical +application header into the carrier digest. For non-genesis carrier round `r`: @@ -234,17 +235,16 @@ virtual. Every embedded consensus vertex has a positive consensus round and an e choice. These conventions avoid making genesis a second, partially authenticated wire format. Weak references are syntax and pacing declarations, not availability assertions. Their target -headers need not be present to authenticate, admit, process, or RBC-deliver the enclosing carrier. +headers need not be present to authenticate, admit, process, or authoritatively deliver the +enclosing carrier. Acknowledgments have one canonical logical order: first the unique maximal suffix shared with `[own_prev] || weak_parents`, then all remaining acknowledgments in their original relative order. The content digest commits to this expanded, suffix-first sequence, while the wire codec stores the shared suffix as an intersection index and retains the order-significant extras. Non-canonical wire -aliases and duplicate acknowledgments are rejected. An honest author creates an acknowledgment -only after the exact target is locally RBC-delivered and its transaction data reconstructs to the -committed root. The acknowledgment becomes usable as data-availability evidence only after its -enclosing carrier is also locally RBC-delivered; an optimistically admitted Byzantine carrier -cannot create inconsistent availability facts at different validators. +aliases and duplicate acknowledgments are rejected. The current standalone runtime emits this +field empty. In particular, a carrier acknowledgment is not an application-DA oracle and cannot +bypass the concrete Core gate described below. `delivery_frontier` has exactly one indexed entry per committee authority. `None` denotes that authority's fixed genesis/empty prefix. A `Some(reference)` entry must name the same authority as @@ -269,24 +269,24 @@ phase batch and optional consensus vertex. It excludes: - recovery or transport metadata. The byte grammar uses a one-byte format version, fixed field markers, big-endian fixed-width -integers, and explicit vector lengths. It does not add a `starfish:block-ref:v2` string to the block -identity. The format byte and unambiguous grammar distinguish this carrier layout; changing the -layout requires a new version and new golden vectors. The canonical identity codec is handwritten; +integers, and explicit vector lengths. It does not add a mutable string domain to the block +identity. Control-only carriers retain the frozen V1 content/wire versions (`01`/`81`); a carrier +with an exact application header uses V2 (`02`/`82`). The canonical identity codec is handwritten; serde or bincode framing is never hashed. -Milestone two freezes the version-one identity grammar as follows. `Ref` is -`author:u16 || carrier_round:u32 || digest:[u8;32]`; every integer is big-endian and every vector -count is `u16`. +`Ref` is `author:u16 || carrier_round:u32 || digest:[u8;32]`; every integer is big-endian and every +vector count is `u16`. The expanded identity grammar is: ```text -00 01 +00 version // 01 control-only, 02 with application header 01 author:u16 02 carrier_round:u32 03 own_prev:Ref 04 weak_count:u16 weak:Ref[] 05 transactions_commitment:[u8;32] +0A canonical_application_header // present exactly for version 02 06 acknowledgment_count:u16 expanded_acknowledgments:Ref[] -07 phase_count:u16 (phase:u8 target:Ref)[] // ECHO=0, READY=1 +07 phase_count:u16 (phase:u8 target:Ref)[] // ECHO=0, READY=1, VOTE=2, ACK=3 08 consensus_present:u8 [ConsensusVertexV1] 09 creation_time_ns:u64 ``` @@ -297,13 +297,13 @@ frontier entries use `0=None` and `1=Some(Ref)`; leader choices use `1=Vote` and reserved for virtual genesis and is rejected on the wire). The canonical transport codec replaces the expanded acknowledgment field with `intersection_start:u16 || extra_count:u16 || extras`, where the intersection is the unique maximal suffix of `[own_prev] || weak_parents`. Decoding expands and -recompresses this field and rejects aliases. To keep the two byte grammars self-describing, this -compressed transport form starts with `00 81`; only expanded identity content starts with `00 01`. +recompresses this field and rejects aliases. The compressed transport versions set the high bit: +`81` for control-only V1 and `82` for application-bearing V2. -Version one caps canonical carrier content at 4 MiB, weak and strong parents at the committee size, +The codec caps canonical carrier content at 4 MiB, weak and strong parents at the committee size, the frontier at exactly the committee size when projected, and encoded phase batches at -`min(4n, 2048)`. The `4n` bound gives two times the expected `2n` steady-state phase arrival rate; -the scheduler still needs the fair-prefix and active-window rules described in Section 8.3. +`min(6n, 2048)`. Six entries per authority leave bounded spillover above the four-phase steady-state +arrival rate; the scheduler limitations are described in Section 8.3. Consensus vertices are referenced by their exact enclosing `BlockReference` plus their declared `consensus_round`. Because there is at most one consensus vertex per carrier, that pair identifies @@ -331,7 +331,7 @@ In MAC mode, author `A` computes entry `q` for recipient `Q_q` over a fixed-widt binds at least: ```text -STARFISH_RBC_DAG_V1 +STARFISH_RBC_DAG_V2 carrier-authentication kind and scheme protocol_instance committee_id @@ -341,9 +341,9 @@ carrier_round canonical carrier content digest ``` -The full vector accompanies every normally disseminated MAC carrier in version one, including -relayed carriers. A receiver verifies only the entry at its own committee index. It neither verifies -nor vouches for the remaining entries. +The full vector accompanies every normally disseminated MAC carrier in the current prototype, +including relayed carriers. A receiver verifies only the entry at its own committee index. It +neither verifies nor vouches for the remaining entries. A carrier received directly from its author and the same carrier received through a relay are both authentication-eligible when the local entry verifies. This is receiver-specific transferable @@ -353,13 +353,14 @@ signature and provides no non-repudiation. The vector is deliberately not an RBC value and has no consistency invariant. A Byzantine author may attach different vectors to the same content reference, including a vector with a valid tag for one recipient and garbage for another. Correctness therefore depends only on the local entry and on -the embedded Bracha protocol, never on agreement about the vector bytes. +the embedded four-phase protocol, never on agreement about the vector bytes. Each node persists one exact vector variant with its carrier for restart and relay. The preference order is locally generated, directly author-received, then first relayed variant with a valid local -entry. Version one does not merge unverified entries from different vectors. Header recovery after -authenticated quorum phase evidence may return canonical content without a vector; that recovery -can unblock RBC delivery but does not create optimistic carrier admission. +entry. The implementation does not merge unverified entries from different vectors. Exact carrier +recovery after phase evidence may return canonical content without a vector; that recovery can +unblock phase progress, authoritative delivery, and READY certification, but it does not create +authenticated carrier admission or fast-clock stake. Public-signature modes use the same context-bound carrier statement without a recipient field and the same embedded RBC/consensus logic. They exist for controlled performance comparison, not as @@ -379,14 +380,23 @@ Authenticated CarrierAdmitted Candidate && Authenticated && accepted by the carrier admission window -Delivered - local Bracha instance reached Q READY and pinned matching canonical content +AuthoritativeDelivered + exact content is locally fixed, authenticated from a necessarily honest author, supported by + O author-excluding ECHO stake, or certified by Q READY + +ReadyCertified + Q READY stake names the exact retained content; recorded independently even if fast delivery + happened earlier + +DataAvailable + control-only carrier, or Core has materialized the exact application block and verified its + committed payload PrefixClosed - Delivered && DataAvailable && exact own_prev prefix is closed + AuthoritativeDelivered && DataAvailable && exact own_prev prefix is closed VertexProjected (orthogonal to the carrier lifecycle) - this carrier's optional consensus vertex is eligible in the certified projection + this carrier's optional consensus vertex is eligible in the authoritative projection Included a committed Starfish anchor frontier names this carrier in its deterministic delta @@ -395,9 +405,11 @@ Ordered the complete included delta is available and the carrier has been deterministically output ``` -`Candidate` alone permits bounded staging and digest-based recovery. `CarrierAdmitted` permits -immediate phase-batch processing and fast-pacemaker counting. `Delivered` permits phase replay even -at a node whose author MAC entry was poisoned. `PrefixClosed` permits frontier inclusion. +`Candidate` alone permits bounded staging and exact-reference recovery. `CarrierAdmitted` permits +immediate phase-batch processing and fast-pacemaker counting. `AuthoritativeDelivered` permits +phase replay even at a node whose author MAC entry was poisoned. `ReadyCertified` is useful audit +and fallback evidence but is not a second output gate after a valid fast delivery. +`PrefixClosed` permits frontier inclusion. `VertexProjected` alone permits the optional vertex to supply Starfish vote/certifier/leader evidence. It is not a later state of every carrier: a carrier with no eligible optional vertex may still become prefix-closed, included by another anchor's frontier, and ordered. @@ -409,17 +421,17 @@ unauthenticated candidate advance the fast clock. | Consumer | Required local authority | |---|---| | Header retention/recovery | `Candidate` | -| Process embedded RBC statements | `CarrierAdmitted`, or `Delivered` for replay | +| Process embedded RBC statements | `CarrierAdmitted`, or `AuthoritativeDelivered` for replay | | Fast carrier clock | `CarrierAdmitted` | -| Transaction/shard synchronization | `Candidate` | -| Count a data-availability acknowledgment | `Delivered` enclosing carrier | +| Exact carrier recovery | requested `Candidate` whose bytes recompute the target reference | +| Accept application payload | carrier-authorized header plus commitment-verified bytes | +| Application data-availability fact | concrete, data-available Core block only | | Delivery frontier | `PrefixClosed` target carrier | | Starfish QC, skip, and anchor commit | `VertexProjected` consensus vertex | -| Application payload output | `Included` delta, with every member delivered/data-available | +| Application payload output | cumulative `Included` delta, with every member authoritatively delivered and application-data-available | -The initial implementation should keep fast-pacemaker counting exactly at `CarrierAdmitted`; using -`Delivered` as an additional stronger pacing input can be added only if the executable model shows -that it cannot change the sequential slot accounting. +Fast-pacemaker counting is exactly `CarrierAdmitted`. Authoritative delivery is intentionally not a +second pacing input because it would change the sequential per-author slot accounting. ## 7. Fast carrier pacemaker @@ -434,23 +446,28 @@ author, but each author contributes stake once. A validator advances from carrie 1. its own carrier at round `r` has been fixed and persisted; and 2. it has admitted distinct-author carrier stake `Q` at round `r`. -Future carriers are bounded and buffered; they do not skip missing local rounds. The next local -carrier records the selected quorum as `{ own_prev } union weak_parents`. A missing weak-parent body -never blocks the next carrier or any later consensus action. +The executable prototype admits authenticated carriers at most two rounds ahead of the local open +round and retains canonical unsolicited carrier content up to 64 rounds ahead. Authenticated +retained carriers may later be promoted sequentially; candidate-only content retains no admission +authority. +Farther unsolicited values are discarded by the benchmark resource guard. Buffered carriers never +skip missing local rounds. These values are engineering bounds, not a proof-safe asynchronous +retirement rule. The next local carrier records the selected quorum as +`{ own_prev } union weak_parents`; a missing weak-parent body never blocks later action. This clock replaces the current `starfish-rbc` rule that requires a quorum of RBC-clean previous round headers before proposal. It does not make admitted carriers consensus votes. Leader/vote/skip waiting conditions move to the independent consensus projection and cannot block the creation of a -carrier needed to transport ECHO or READY. +carrier needed to transport ECHO, VOTE, ACK, or READY. Honest validators emit empty control heartbeats when they have no application transactions. -Without heartbeats, low load can stop the ECHO/READY waves and violate RBC liveness. Every carrier, +Without heartbeats, low load can stop the four-phase waves and violate RBC liveness. Every carrier, including an empty heartbeat, is itself an RBC value and has an authenticator sidecar. The first prototype retains the current run's carrier/RBC state and rate-limits carrier creation. A production design needs a proved runahead and backpressure rule. A hard carrier/consensus skew cap must not suppress control heartbeats, because those heartbeats may be exactly what allows the -certified frontier to catch up. +authoritative projection and committed frontier to catch up. ## 8. Embedded all-carrier reliable broadcast @@ -461,89 +478,106 @@ RbcSlot = (protocol_instance, committee_id, carrier_author, carrier_round) RbcValue = BlockReference ``` -Authenticating the carrier is INIT for that value. The local author records its own ECHO when it -atomically fixes and persists the carrier. An honest non-author that first admits a value queues one -ECHO for that slot. +Receiver authentication is the admission capability for that exact value. A non-author that first +admits a value durably locks and queues ECHO. The target author never contributes ECHO, VOTE, or ACK +stake, and target-authored statements in those phases are ignored. Locally fixing a carrier is an +authoritative delivery predicate for its creator, but it does not manufacture author stake in an +author-excluding certificate. -INIT is not silently counted as the author's ECHO at remote validators. The author's recorded local -ECHO is queued into a later carrier like every other phase action; it counts locally immediately and -remotely only after the enclosing carrier is admitted or delivered. This preserves the standard -quorum accounting without excluding the broadcaster's stake. +For a potentially Byzantine target (`a <= F`), the reducer applies these weighted transitions: -Local phase actions are inserted into the next possible carrier's `phase_batch`. The enclosing -carrier's authentication makes its author the phase sender. For each target slot: +```text +admit exact content -> ECHO +ECHO stake >= M -> VOTE +ECHO stake >= C or VOTE stake >= C -> ACK +ACK stake >= C or READY stake >= V -> READY +ECHO stake >= O -> authoritative optimistic delivery +READY stake >= Q -> independent READY certification +``` -- an authority emits at most one ECHO; -- an authority emits at most one READY; -- ECHO and READY choices may differ; -- `Q` ECHO stake creates a READY obligation; -- `V` READY stake creates a READY obligation; and -- `Q` READY stake plus pinned matching content locally delivers the exact carrier. +ECHO, VOTE, and ACK use distinct-author stake outside the target author; READY uses the full +committee. VOTE does not require local carrier admission, only the latched ECHO evidence and exact +retained target content. ACK may arise from `C` ECHO or `C` VOTE and does not require the node to +have sent VOTE. READY may arise from `C` ACK or the standard `V` READY amplification path. Local +phase evidence counts immediately after its durable lock, before later carrier dissemination. -A READY obligation is not yet a local READY. If the target content is absent, the validator first -recovers it from authenticated ECHO/READY authors, validates the exact reference, and durably pins -it. Only then does it persist the slot-global READY lock, count its local READY, and enqueue the -statement. Consequently every honest READY author is a real content holder. A quorum trigger remains -latched while recovery is pending. +If `a > F`, the target author is necessarily honest. A valid receiver-bound authenticator on the +exact author value is therefore an immediate authoritative delivery predicate. The locally fixed +high-stake author also seeds READY; because its stake is at least `V`, the ordinary amplification +and `Q` certification path still completes independently. -Local ECHO and READY count toward their thresholds before network dissemination. Evidence is -tracked per candidate, while local send and delivery locks are slot-global. Each remote authority -contributes stake at most once per phase and target slot; exact replay is idempotent and a later -equivocation is ignored before allocating another candidate. +Reaching `O` is authoritative delivery, not a speculative hint. The reducer pins the exact carrier, +advances its delivered prefix when the DA and predecessor conditions hold, and may use its optional +vertex in the authoritative projection. `Q` READY records the slower certificate even if the same +value already delivered through the fast predicate. A certificate reached first also satisfies the +delivery predicate. No threshold over a bare digest can expose a phase or deliver a placeholder. + +For every local slot, ECHO, VOTE, ACK, READY, delivery, and READY certification each lock at most +one target; different phases may safely name different targets. For every remote sender, the node +also records at most one target per phase and slot. Exact replay is idempotent, and a later +same-sender equivocation is ignored before it can add stake or allocate a second phase choice. ### 8.1 Processing rule -After canonical validation and outer authentication, a receiver processes the phase batch in its -canonical encoded order immediately. It must not wait for the enclosing carrier itself to be RBC -delivered, data-available, dependency-closed, or projected. Waiting would create a recursion: a -carrier's controls are required to deliver earlier carriers, while those earlier deliveries may be -required to create the next consensus vertex. +After canonical validation and admission of the outer authenticator, a receiver processes the +phase batch in canonical encoded order immediately. An authenticated carrier beyond the two-round +admission window is retained but neither contributes pacemaker stake nor creates ECHO; its batch +gains authority only after sequential promotion or an independent authoritative-delivery predicate. +A retained candidate alone has no phase authority. Once authorized, processing does not wait for +the enclosing carrier's READY certificate, application DA, prefix closure, or projection. Waiting +would create a recursion because its controls are needed to deliver earlier carriers. If the local carrier authenticator is missing or invalid, its phase batch is not processed on -candidate receipt. If that exact outer carrier later becomes locally RBC-delivered, the stored batch -may be replayed under the local delivery capability. Thus poisoned vector entries delay optimistic -admission but cannot permanently suppress controls selected by RBC. +candidate receipt. If that exact outer carrier later becomes authoritatively delivered, the stored +batch is replayed under that delivery capability. Thus poisoned vector entries delay optimistic +admission but cannot permanently suppress controls selected by the four-phase protocol. Direct +phase messages are never an alternative authority source. Phase targets are strictly older carrier rounds, making a single carrier's replay acyclic. Local arrival order between different authenticated carriers is still observable and can affect which -Byzantine equivocation encounters a slot-global guard first. Recovery must replay the persisted -ingress journal, never reconstruct choices by sorting carriers after a restart. +Byzantine equivocation encounters a slot-global guard first. The ordered journal persists each +batch entry and its replay cursor; recovery must replay that order, never reconstruct choices by +sorting carriers after a restart. ### 8.2 Header recovery -An honest ECHO or READY author must retain the target carrier content. A validator that observes a -threshold before receiving the target requests it from several recorded phase authors. Recovery -content is accepted only when canonical validation recomputes the requested reference. - -Recovery request/response remains an out-of-band data-transfer optimization in the first -prototype. It is not quorum testimony and does not change the on-DAG phase transcript. A `Q` ECHO -set contains honest holders, and a `V` READY set contains at least one honest holder, so retrying -authenticated holders eventually obtains the value after GST. +An honest ECHO, VOTE, ACK, or READY author retains the exact target carrier content before exposing +that phase. A validator may therefore latch threshold evidence before it has the bytes, record the +union of phase senders as candidate holders, and request the target from those holders. Recovery +content is accepted only when canonical decoding recomputes the requested `BlockReference` and the +context/committee checks succeed. Retention precedes any new local phase lock. + +Recovery request/response is out-of-band byte transfer, not quorum testimony, admission, or a new +phase. A valid response can satisfy an already allocated evidence obligation but cannot create one. +The prototype retries recorded holders after GST. Its separate carrier catch-up mechanism requests +one exact `(author, round)` at a time and serves only retained locally authored outbound bytes; it +does not transfer ranges, certificates, checkpoints, committed observer history, or arbitrary late +state. ### 8.3 Batching and fairness Phase batches are bounded. The encoded order is preserved and processed as an authenticated log; two different orders intentionally identify different carriers. A deterministic fair queue must prevent Byzantine traffic for one slot -from starving honest ECHO/READY actions for other slots. In steady state, one authority can owe one -ECHO and one READY for each of `n` previous-round carriers, so `2n` is the expected arrival rate and -not a safe capacity. The executable model retains an unbounded pending FIFO and drains the first -`4n` statements eligible for the carrier being built (capped by the version-one codec limit of +from starving honest ECHO/VOTE/ACK/READY actions for other slots. In steady state, one authority can +owe up to four actions for each of `n` previous carriers. The executable model retains an unbounded +pending FIFO and drains the first `6n` statements eligible for the carrier being built (capped at 2,048 statements). A temporarily ineligible future-round statement remains in its stable queue position but does not block older eligible work behind it. This exercises backlog, runahead, and batching without pretending to solve adversarial fairness. A bounded runtime must use a fair -per-slot scheduler, reserve strictly more than `2n` statements per carrier, and enforce an +per-slot scheduler, reserve spillover above the four-phase arrival rate, and enforce an active-slot window so delayed work drains instead of remaining at permanent saturation. -## 9. Certified consensus vertices +## 9. Authoritative consensus vertices A carrier contains zero or one `ConsensusVertexV1`. The carrier remains valid and pace-eligible if -the optional vertex is malformed relative to local certified state; only the optional vertex is -excluded from the consensus projection. +the optional vertex is malformed relative to authoritative projection state; only the optional +vertex is excluded from the consensus projection. A consensus vertex authored by `A` at consensus round `c > 0` is eligible only when: -1. its enclosing carrier is locally RBC-delivered; +1. its enclosing carrier is authoritatively delivered by a local-fixed, honest-author, + `O`-ECHO, or `Q`-READY predicate; 2. its enclosing carrier's transaction data is available and it closes `A`'s carrier prefix as defined in Section 10; 3. its strong parents name distinct-author eligible consensus vertices at exactly `c - 1` whose @@ -569,7 +603,8 @@ strong parents block only this optional vertex. They never block the enclosing c batch, the fast clock, or later honest RBC progress. The consensus pacemaker preserves Starfish's separate advance and creation conditions, evaluated -only over eligible consensus vertices: +only over eligible consensus vertices. A `Q`-READY certificate is not additionally required after +an authoritative optimistic delivery: - **A1:** advance from `c - 1` to `c` after eligible distinct-author stake `Q` at `c - 1`; - **A2:** do not advance until the local consensus vertex at `c - 1` has been fixed; @@ -607,7 +642,8 @@ choice—must resolve Byzantine conflicts. ## 10. Closed delivery prefixes and frontiers -RBC delivery alone is not a compact availability proof for a Byzantine author's later carrier. A +Authoritative carrier delivery alone is not application data availability and is not a compact +availability proof for a Byzantine author's later carrier. A Byzantine author may deliver round `r` with an `own_prev` that names an unavailable fork at `r - 1`. Therefore a frontier component is a contiguous exact prefix, not simply the highest delivered round. @@ -615,8 +651,9 @@ delivered round. For authority `A`, begin at its fixed genesis/empty prefix. A carrier `(A, r, R)` extends the local closed prefix only when: -- `R` is locally RBC-delivered; -- its transaction data satisfies the existing Starfish availability predicate; +- `R` is authoritatively delivered; +- for an application carrier, Core has materialized the exact application block and its committed + transaction data satisfies the existing Starfish availability predicate; - `r` is exactly one more than the current prefix round; and - `R.own_prev` equals the exact current prefix tip. @@ -633,23 +670,24 @@ ensures that committed frontiers never regress or switch Byzantine forks. The containing carrier cannot name itself in its encoded frontier. For an eligible consensus vertex, its declared author component must equal its carrier's `own_prev` prefix tip. Once the -enclosing carrier is delivered and data-available, its **effective frontier** replaces that one -component with the enclosing carrier. This makes a committed anchor's own application payload -eligible without waiting for a later anchor while preserving exact prefix continuity. +enclosing carrier is authoritatively delivered and data-available, its **effective frontier** +replaces that one component with the enclosing carrier. This makes a committed anchor's own +application payload eligible without waiting for a later anchor while preserving exact prefix +continuity. The liveness target is deliberately precise: -> Every honest carrier that RBC-delivers and becomes data-available eventually appears in a +> Every honest carrier that authoritatively delivers and becomes data-available eventually appears in a > committed effective-frontier delta. No guarantee is made for a malformed or permanently off-prefix Byzantine carrier. Guaranteeing all -RBC-delivered Byzantine forks would require an antichain or sparse exception structure rather than -one compact prefix tip per authority. +authoritatively delivered Byzantine forks would require an antichain or sparse exception structure +rather than one compact prefix tip per authority. ## 11. Starfish certification, commit, and skip Starfish's logical leader schedule and commit rules run over eligible consensus vertices only. -Carrier admission, weak parents, phase targets, candidate headers, and merely delivered carriers +Carrier admission, weak parents, phase targets, candidate headers, and merely retained carriers cannot act as voters, certifiers, leaders, non-votes, or reachability evidence. For a scheduled leader slot at consensus round `c`, every eligible voter publishes one immutable @@ -681,33 +719,35 @@ Skipping a Byzantine leader role discards only that optional consensus value. It the enclosing application carrier. If that carrier later becomes part of a closed prefix, a later committed frontier orders its payload. -Every consensus consumer in the current Starfish committer must be audited for the new type -boundary: voter caches, leader support, potential certificates, direct/indirect decisions, -reachability, and the linearizer must reject non-projected carrier facts. Data-availability -acknowledgments are the deliberate exception: they become usable when their enclosing carrier is -RBC-delivered, which breaks a projection/availability circularity while still excluding merely -optimistic evidence. +Every consensus consumer enforces this type boundary: voter caches, leader support, potential +certificates, direct/indirect decisions, reachability, and the linearizer reject non-projected +carrier facts. Application data availability is not inferred from this projection or from carrier +acknowledgment references; it enters only through the typed Core materialization callback. ## 12. Frontier-delta linearization -Let `F_k` be the effective frontier carried by committed anchor `A_k`, and let `Closure(F_k)` be the -union of the exact per-author self-chain prefixes named by `F_k`. Maintain: +Let `F_k` be the effective frontier carried by committed anchor `A_k`, `J_k` the cumulative joined +committed frontier, and `Closure(F)` the union of the exact per-author self-chain prefixes named by +`F`. Maintain: ```text -C_0 = fixed genesis carriers -C_k = C_(k-1) union Closure(F_k) -Delta = C_k \ C_(k-1) +J_0 = [None; committee_size] +J_k = componentwise_exact_join(J_(k-1), F_k) +Delta = Closure(J_k) \ Closure(J_(k-1)) ``` -Before outputting `Delta`, a validator waits until every exact member is locally RBC-delivered and -data-available. RBC totality and erasure-coded recovery supply missing content for honest committed -frontiers. +The join compares exact self-chain lineage, not round numbers. It accumulates advances from +concurrent committed anchors and prevents a later partial frontier from erasing or regressing a +component already committed. Before outputting `Delta`, a validator requires every exact carrier +to be authoritatively delivered and every application member to pass the concrete Core DA gate. +Exact carrier recovery and the dedicated verified-payload path supply missing material. All validators deterministically order the same delta by `(carrier_round, author, content_digest)`. Because a closed author prefix advances by exactly one carrier round, this key already preserves mandatory `own_prev` order. -Weak parents, strong consensus edges, optional-vertex projection time, ECHO/READY target references, +Weak parents, strong consensus edges, optional-vertex projection time, +ECHO/VOTE/ACK/READY target references, recovery provenance, and MAC-vector variants never constrain application payload ordering. Strong edges order consensus decisions and dominate frontiers, but a late-projecting optional vertex must not retroactively add an edge between payloads already output. This fixed ordering also ensures that @@ -719,15 +759,18 @@ design removes. In an all-honest synchronous interval, batching can realize this conceptual schedule: ```text -t = 0 carrier k contains a new application header (RBC INIT) -t = delta carrier k+1 contains ECHOs for k -t = 2delta carrier k+2 contains READYs for k -t = 3delta carrier k is RBC-delivered; a later carrier may project new consensus work +t = 0 carrier k authenticates an exact application header; non-authors lock ECHO +t = delta ECHO stake reaches O: authoritative delivery; VOTE/ACK obligations are queued +t = 2delta ACK stake reaches C: READY is queued +t = 3delta READY stake reaches Q: the independent certificate is recorded ``` -The embedded design does not make Bracha RBC require fewer communication delays than the direct -baseline. Its performance hypothesis is that carrier batching reduces frames, scheduling work, and -duplicated control metadata while the fast carrier clock overlaps certification with dissemination. +The `O` fast path deliberately makes authoritative delivery available after the ECHO wave instead +of waiting for the READY certificate. The VOTE/ACK/READY path remains necessary for convergence, +totality, and independent certification under adverse schedules. For `a > F`, receiver-authenticated +exact content takes the honest-author fast branch even before the ECHO threshold. Application +output still waits for DA, logical consensus, and the cumulative committed frontier, so the RBC +delivery schedule is not itself a transaction-latency claim. Implementation ordering is latency-critical. On carrier ingress, authenticate, apply its phase batch, execute newly enabled delivery/prefix/projection transitions, and only then decide what the @@ -746,8 +789,8 @@ An authoritative implementation must persist proof-critical choices before expos 1. journal typed authenticated inbound provenance, exact bytes, and its local ingress sequence; 2. before fixing a local slot, construct and persist the typed candidate plus its exact canonical carrier bytes and reference; -3. persist local ECHO, READY, explicit leader-choice, delivery, carrier-slot, and consensus-slot - locks that match that retained candidate (recovered content is likewise retained before READY); +3. persist local ECHO, VOTE, ACK, READY, delivery-promise, READY-certificate, explicit + leader-choice, carrier-slot, and consensus-slot locks that match retained exact content; 4. persist the exact authentication sidecar and an outbound-exposure marker, and only then send the carrier; and 5. after restart, replay the journal in recorded order and retransmit the identical carrier and @@ -760,35 +803,45 @@ until every lock encoded by that carrier is durable. Every persisted slot, candidate, lifecycle predicate, journal entry, and outbound-carrier key is namespaced by both `protocol_instance` and `committee_id`; storage from another run or committee -cannot satisfy a local lock or quorum. +cannot satisfy a local lock or quorum. The current authoritative storage path is additionally +separated as autonomous WAL V4, and raw journal records start with `SRD5`. V4 prevents traces from +the older planning-only promise semantics from being reinterpreted as authoritative fast delivery; +`SRD5` prevents older record layouts from decoding as the current ECHO/VOTE/ACK/READY journal. Hash-sorting recovered carriers is not a valid reconstruction rule. Byzantine equivocation can make arrival order determine which value a local slot-global guard selects, and a different restart order could make one honest authority appear to send conflicting phases. +The Core store has a bounded latest-frontier receipt shape and can atomically persist an application +commit with that receipt, but that alone is not a complete actor-to-Core recovery handshake. Until +startup re-emits exactly the actor-WAL frontier suffix newer than Core's durable cursor, applies it +before the `Ready`/application-production barrier, and rejects a cursor the actor cannot reconcile, +a crash between actor durability and Core application remains a fail-stop boundary. Actor replay +must not be described as exactly-once output without that composed contract. + The proof model retains all proof-critical carrier, phase, prefix, and consensus state for the run. -The milestone-three shadow bounds newly arriving unsolicited content to a fixed recent-round -window solely to keep a faulty peer from growing an observational benchmark process without limit. -This is not a protocol-safe retirement rule: an honest INIT may be delayed longer than that under -asynchrony. Recovery of an exact already-requested value is exempt. Before authoritative garbage -collection is enabled, the design needs a common certified or committed retirement watermark that -preserves: - -- pending Bracha totality and header recovery; +The executable runtime admits at lookahead `2` and bounds newly arriving unsolicited content at +lookahead `64` solely to keep a faulty or descheduled peer from growing the benchmark process +without limit. This is not a protocol-safe retirement rule: an honest carrier may be delayed more +than 64 rounds under asynchrony. Recovery of an exact already-requested value is exempt. Before +authoritative garbage collection is enabled, the design needs a common certified or committed +retirement watermark that preserves: + +- pending four-phase totality and exact-content recovery; - exact self-prefix expansion from the last committed frontier; - committed-anchor reconstruction for a late validator; and - deterministic replay of local locks. Resource bounds still required before authoritative deployment include a proof-safe future and -retirement window, per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, -a bounded payload runahead policy, and checkpointed disk-backed recovery. Shadow input and output -channels are bounded and shed observational work instead of backpressuring direct consensus, but -the reference reducer's retained history and per-transition validation are not yet bounded-runtime -architecture. Resource exhaustion is excluded from the initial proof model and must be measured in -the prototype. Any run in which work is shed is invalid for direct/shadow comparison; +retirement window, per-peer candidate caps, a fair bounded phase backlog, bounded peer/network +bridges, a rate-limited control heartbeat, a bounded payload runahead policy, and checkpointed +disk-backed recovery. The actor's primary ingress is bounded, but not every bridge, per-peer outbox, +or retained reducer collection is yet bounded end to end. Resource exhaustion is excluded from the +initial proof model and must be measured in the prototype. Any run in which work is shed is invalid +for direct/shadow comparison; `starfish_rbc_dag_shadow_comparison_valid` must remain `1` for the entire measured interval. A live pipeline does not have equal cumulative direct and shadow delivery counters at an arbitrary -instant: embedded ECHO/READY normally leaves a short shadow tail. Benchmark verification therefore +instant: the embedded four-phase protocol normally leaves a short shadow tail. Benchmark verification therefore requires monotone nonzero direct, shadow, and paired-match progress, no conflict outcome, and bounds both the current unpaired slots (`<= 4n` per validator) and the oldest unpaired round lag against the newest current-process observation (`<= 4`). These are empirical benchmark coverage guards, not @@ -803,8 +856,10 @@ at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slo exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, healthy proactive rounds receive a repair grace period, and exact synchronization is rate-limited -per peer. Requested historical slots remain recoverable beyond the benchmark-only unsolicited -retention window. +per peer. Exact synchronization transfers only one requested `(author, round)` and only from the +author's retained local outbound map. It has no range response, certified checkpoint, observer +history, or bounded-suffix state-transfer protocol; a sufficiently late or fresh node cannot be +reconstructed by this mechanism alone. Autonomous benchmark validity is separate from delivery comparison validity. `starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and local-carrier counters @@ -821,35 +876,42 @@ The design is not complete until at least the following claims are proved or fal 1. **Receiver-authentication integrity.** An honest receiver admits a carrier attributed to an honest author only if that author created the public proof or the receiver's MAC entry. A MAC is not public non-repudiation, and a Byzantine endpoint knows its own pairwise key. -2. **RBC agreement and integrity.** Slot-global ECHO/READY locks, quorum intersection, and exact - value binding prevent two conflicting carrier values from being delivered by honest validators. -3. **RBC totality.** If one honest validator delivers a value, heartbeats, READY amplification, and - holder recovery cause every honest validator eventually to deliver the same value. -4. **Optimistic isolation.** Carrier admission can change only fast pacing and RBC processing; it - cannot alter a QC, leader decision, skip, commit, acknowledgment certificate, or output order. -5. **Weak-edge non-poisoning.** A missing or equivocating weak parent cannot block delivery, +2. **Four-phase agreement and integrity.** Target-author exclusion, the `M/C/O/V/Q` intersections, + slot-global ECHO/VOTE/ACK/READY locks, per-sender phase locks, and exact value binding prevent + conflicting authoritative deliveries at honest validators. +3. **Fast-predicate safety.** `a > F` really implies an honest author, and an `O = M + b` ECHO set + contains enough honest non-author support to make the value unique and force the fallback to + converge on it. +4. **RBC totality and certification.** If one honest validator authoritatively delivers a value, + VOTE/ACK convergence, READY amplification, heartbeats, and exact holder recovery cause every + honest validator to deliver it and eventually record the same `Q`-READY certificate. +5. **Admission isolation.** Mere authentication/admission can change fast pacing and phase replay, + but cannot alter a QC, leader decision, skip, commit, or output. Only a proved authoritative + delivery predicate plus prefix/DA/projection gates crosses that boundary. +6. **Weak-edge non-poisoning.** A missing or equivocating weak parent cannot block delivery, projection of unrelated honest vertices, or application ordering. -6. **Consensus-slot uniqueness.** Honest validators create/vote once per +7. **Consensus-slot uniqueness.** Honest validators create/vote once per `(author, consensus_round)`, and Byzantine conflicts cannot both acquire honest quorum support. -7. **Prefix comparability.** Every accepted frontier component is an exact extension of its strong - ancestors and of every earlier committed component. -8. **Projection safety.** Erasing weak edges and optional consensus metadata that is not +8. **Prefix and join comparability.** Every accepted frontier component is an exact extension of + its strong ancestors, and every cumulative committed join is monotone on exact lineage. +9. **Projection safety.** Erasing weak edges and optional consensus metadata that is not `VertexProjected` leaves a valid execution of the Starfish commit/skip rules over immutable strong edges; it does not erase otherwise orderable carrier payloads. -9. **Deterministic ordering.** Equal committed anchors imply equal frontier closures, deltas, and - transaction order at all honest validators. -10. **Data availability.** No carrier enters an output delta until its committed transaction root - can be reconstructed and verified. +10. **Deterministic ordering.** Equal committed-anchor histories imply equal cumulative frontier + joins, closures, deltas, and transaction order at all honest validators. +11. **Data availability.** No application enters an output delta until Core contains the concrete + block and its committed transaction root has been reconstructed and verified. ## 16. Liveness obligations Under partial synchrony and fair processing, the design must establish: -1. `Q` honest authors continually create authenticated carriers after GST, so the sequential fast - clock advances without Byzantine participation. -2. Empty heartbeat carriers drain every honest ECHO/READY backlog even when application load is +1. Honest authors of aggregate stake at least `Q` continually create authenticated carriers after + GST, so the sequential fast clock advances without Byzantine participation. +2. Empty heartbeat carriers drain every honest ECHO/VOTE/ACK/READY backlog even when application load is zero. -3. Every honest carrier is RBC-delivered at every honest validator. +3. Every honest carrier is authoritatively delivered and eventually `Q`-READY certified at every + honest validator. 4. Existing Starfish data availability eventually closes every honest author's exact carrier prefix. 5. Honest consensus vertices with quorum strong parents continue to appear despite arbitrary @@ -857,46 +919,56 @@ Under partial synchrony and fair processing, the design must establish: 6. The projected Starfish pacemaker eventually commits infinitely many honest anchors. 7. Honest frontier construction is fair: every newly closed honest carrier prefix is eventually included in a committed frontier. -8. Waiting for a committed delta cannot block forever because every named exact carrier is already - RBC-delivered and data-available by frontier eligibility. +8. Waiting for a committed delta cannot block forever for honest applications because every named + exact carrier is already authoritatively delivered and the concrete application DA condition is + part of frontier eligibility. The guaranteed payload-liveness statement covers every honest on-prefix carrier. Selectively disseminated, malformed, or off-prefix Byzantine carriers may be ignored. ## 17. Required executable tests -Milestone two begins with an isolated deterministic model, not production network wiring. At -minimum it must cover: +The executable model and composed runtime tests should cover at minimum: - `n = 4, f = 1` and `n = 7, f = 2` all-honest progress; -- split Byzantine INIT values and receiver-selective poisoned vector entries; +- split Byzantine author values and receiver-selective poisoned vector entries; - valid relayed local MAC entries and invalid vector variants; -- ECHO/READY equivocation, replay, reordering, and evidence-before-header recovery; +- weighted and unequal-stake `F/a/U/b/M/C/O/V/Q` threshold goldens, including `a > F`; +- ECHO/VOTE/ACK/READY local locks, per-sender replay/equivocation, ordered batch replay, and restart; +- `M` ECHO to VOTE, `C` ECHO-or-VOTE to ACK, `C` ACK to READY, `O` authoritative delivery, and + independent `Q`-READY certification; +- evidence-before-content recovery from phase holders, including VOTE/ACK without local admission; - zero application load with heartbeat-only RBC completion; -- future carriers that cannot jump the local sequential clock; +- two-round admission, 64-round authenticated retention, and future carriers that cannot jump the + local sequential clock; - `f` permanently missing weak parents without blocking honest carrier or consensus progress; -- a delivered Byzantine carrier above an unavailable self-chain gap; +- an authoritatively delivered Byzantine carrier above an unavailable self-chain gap; - conflicting Byzantine consensus vertices in one logical slot; - explicit vote/no-vote conflicts and direct plus indirect commit/skip; - frontier fork, regression, and strong-parent dominance rejection; -- equal committed anchors producing byte-identical output deltas; -- delayed data availability followed by eventual prefix inclusion; +- concurrent committed anchors producing the same cumulative joined frontier and byte-identical + output deltas without regression; +- delayed concrete Core data availability followed by eventual prefix inclusion, with raw payload + receipt unable to bypass the gate; - crash points before and after each persisted lock and outbound-carrier write; and -- persisted shadow-actor restart with byte-identical retransmission against an identical recovered - direct-header history, bounded overload, poisoned-tag candidate retention, exact recovery, and - paired delivery observations against the current direct RBC kernel. Full validator restart is - excluded until the authoritative direct-RBC locks are durable; and +- crash points before and after Core frontier application, including atomic receipt/commit, + exact-replay idempotence, conflicting/stale cursor rejection, suffix replay before `Ready`, and + explicit failure of the buffered-WAL crash-safety case; and +- persisted actor restart with byte-identical local retransmission, poisoned-tag candidate + retention, exact recovery, and no phase/output exposure before its matching durable locks; and - autonomous actor progress at `n = 4` and `n = 7`, no steady-state repair polling on healthy proactive rounds, exact-slot synchronization with idempotent late responses and per-peer rate limiting, multi-round convergence after a validator falls behind, control-only WAL reopen, - distinct authentication namespace, and an integration check that direct Starfish-RBC continues - committing while the observational carrier clock advances; and -- composed frontier-authority runs in which every exact application header is embedded-RBC + distinct authentication namespace, V4/`SRD5` stale-trace rejection, and bounded exact-slot sync + that does not pretend to be checkpoint transfer; and +- composed frontier-authority runs in which every exact application header is authoritatively delivered, all honest nodes release the same deterministic application order without duplicates, - the legacy committer is disabled, and the frontier/application/WAL progress gates remain valid. + the direct INIT/phase/batch/pull paths and legacy committer have no authority, and the + frontier/application/WAL progress gates remain valid. Property tests should mutate every canonical field and verify carrier-reference binding, while -golden tests freeze the version-one encoding and flat vector length. +golden tests freeze the V1 control and V2 application encodings, append-only phase codes, and flat +vector length. ## 18. Complexity and benchmark plan @@ -909,7 +981,7 @@ The first fair benchmark matrix includes: - Sailfish++ as a certified signature-free comparison. Hold committee, load, transaction size, topology, latency injection, dissemination fanout, duration, -timeouts, and build constant. Report carrier/INIT, vector, ECHO, READY, recovery, transaction/shard, +timeouts, and build constant. Report carrier, vector, ECHO, VOTE, ACK, READY, recovery, payload, and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained state. @@ -917,20 +989,20 @@ state. Batching can reduce the number of separately scheduled RBC control messages, but it does not remove their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout -transport is added. Shadow mode also sends both direct and embedded transcripts. The default -crash-safe reference profile fsyncs each accepted transition and validates through a clone-based -reducer; it is a correctness/replay instrument, not a protocol-performance result. The explicit -buffered-WAL profile keeps the exact framed event path but syncs only on clean shutdown and therefore -cannot be used for crash-safety claims. Benchmark output reports appended and durable WAL work -separately. The clone-based reducer remains intentionally unoptimized until measurement shows it -matters. +transport is added. Comparison mode sends both direct and embedded transcripts; standalone mode +does not. The default crash-safe reference profile applies preflighted transitions in place but +fsyncs every accepted transition and still performs synchronous reducer/storage work, so it is a +correctness/replay instrument rather than a fair latency profile. The explicit buffered-WAL profile +keeps the exact framed event path but syncs only on clean shutdown and therefore cannot support +crash-safety claims. Benchmark output reports appended and durable WAL work separately. A matched 10-validator local sequence on 2026-08-11 used a full 60-second active transaction window, the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and the buffered benchmark WAL. Milestone-five idle carriers use the same resolved 600 ms Push leader timeout as Starfish-RBC; application and encodable phase carriers are immediate. The harness waits through generator warmup, snapshots cumulative counters at the active boundary, and drains final latency -samples. +samples. The table and milestone narrative below predate the current four-phase V4 authority model +and remain historical evidence rather than a current-protocol result. | Profile | Verdict | TPS | Block latency | E2E latency | Outbound | |---|---:|---:|---:|---:|---:| @@ -974,50 +1046,36 @@ toward the roughly 600 ms unsafe Starfish-MAC reference without weakening RBC or ## 19. Contained implementation milestones -Every milestone is committed separately. - -1. **Protocol specification (this document):** lock the two clocks, lifecycle, full-vector sidecar, - embedded Bracha transitions, certified prefix/frontier, commit/skip boundary, proof obligations, - and experiment plan. No protocol code or CLI selector is added. -2. **Canonical codec and executable model (implemented):** isolated carrier, phase, consensus, - frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable - journal models; and deterministic adversarial simulations. No network or existing consensus path - changes. -3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the - current direct `starfish-rbc` service, cache the validated committee/domain identity rather than - re-hashing all public keys per carrier, journal ingress and local locks, and compare embedded - versus direct RBC delivery through current-process paired observations. Direct RBC remains - authoritative; shadow results never affect proposals or commits. The reference WAL/reducer is a - correctness instrument, not yet an interpretable protocol-performance path. -4. **Optimistic carrier clock (implemented, opt-in control shadow):** run a separately namespaced, - control-only heartbeat carrier plane with the distinct authenticated-admission latch, sequential - quorum clock, bounded future buffer, exact-slot synchronization, durable restart, and clock - validity metrics while consensus still uses the current direct baseline. -5. **Authoritative embedded RBC (implemented, opt-in):** encode exact canonical application headers - in version-two carriers, durably reconcile their origins, schedule application/phase carriers - immediately, and remove direct ECHO/READY/delivery authority. Direct INIT remains payload - transport; composed tests assert zero direct ECHO/READY traffic and positive embedded delivery. -6. **Certified consensus projection (implemented):** add optional independently numbered consensus - vertices, quorum strong parents, explicit timeout-bound leader choices, contiguous exact - delivery frontiers, durable slot/choice locks, and a live clean-only direct committer. Malformed - optional vertices do not poison their enclosing carrier. -7. **Frontier linearizer and recovery (implemented within the actor's fail-stop scope):** commit - deterministic frontier deltas, reconstruct prefixes, decisions, and anchors from the ordered - WAL, disable the legacy application committer, and output exact application references once. - Full-validator crash recovery and proof-safe late-node state transfer remain deferred because - the direct payload-transport baseline does not yet persist its own proof-critical RBC state. -8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, - signature Starfish variants, and Sailfish++ before attempting tree dissemination. -9. **Tree dissemination:** distribute vector sub-bundles with redundant routing and a direct timeout - fallback; do not change RBC or consensus semantics. +The historical milestones produced the current bounded prototype: + +1. **Canonical carrier plane (implemented):** V1 control and V2 application carrier identities, + full-vector sidecars, two independent clocks, ordered phase batches, exact recovery, and + deterministic codec/model/journal tests. +2. **Weighted optimistic RBC (implemented):** author-excluding ECHO/VOTE/ACK thresholds, READY + fallback/certification, the high-stake honest-author branch, `O` authoritative delivery, and + durable slot-global locks for every phase. +3. **Standalone authority boundary (implemented, opt-in):** carriers and the dedicated verified + application-payload path are the only ingress authority. The direct INIT/phase/header service, + generic batch/pull path, and legacy application committer are absent or rejected. +4. **Logical consensus and output (implemented):** independently numbered vertices, quorum strong + parents, explicit Vote/NoVote choices, exact prefixes, authoritative projection, Starfish + commit/skip, cumulative joined committed frontiers, and deterministic application deltas. +5. **Durable actor replay (implemented within the documented scope):** V4/`SRD5` WAL replay restores + retained content, phase/delivery/consensus locks, locally authored outbound bytes, and projection + state. This is not a claim of full validator restart or general late-node state transfer. +6. **Remaining production work:** complete the end-to-end proof; bound every queue and retained + collection; add proof-safe checkpoints, state transfer, retirement, and graceful shutdown; then + compare latency with direct `starfish-rbc`, unsafe `starfish-mac`, signature variants, and + Sailfish++ before attempting tree dissemination. ## 20. Decisions intentionally deferred -The following values are not safe to guess in the documentation milestone and must be resolved by -the executable model or measured prototype: +The following production choices remain unresolved and must be proved or measured: -- production maximum future-carrier buffer and payload runahead (the executable model deliberately - uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); +- production maximum future-carrier buffer and payload runahead (the executable prototype keeps + admission lookahead `2`, retains at most `64` future rounds for temporarily descheduled peers, + and discards farther unsolicited carriers before admission/retention; these are benchmark resource + parameters rather than protocol safety constants); - whether the shared Starfish leader-timeout policy needs a separately proved adaptive low-load rule; the prototype intentionally does not introduce a second heartbeat timeout; - a safe state-retirement, garbage-collection, and late-catch-up watermark; @@ -1025,6 +1083,7 @@ the executable model or measured prototype: - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and - the tree topology, redundancy, and fallback timers. -Mixed `starfish-rbc`, `starfish-rbc-dag`, and version-one/version-two deployments must be rejected -by protocol-instance negotiation. The provisional `starfish-rbc-dag` selector is added only after -the codec/model milestone establishes a distinct stable version. +Mixed direct/standalone or incompatible carrier/WAL deployments must be rejected. The current code +has fail-closed protocol/storage namespaces but no feature handshake, so homogeneous configuration +is an operational precondition. A dedicated `starfish-rbc-dag` selector should be added only with +explicit capability/version negotiation. From c779699343ba07107966d0482127bdc8fd67cde4 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:50:33 +0200 Subject: [PATCH 37/62] Fix RBC-DAG ordered frontier commitment --- .../src/starfish_rbc_dag/projection.rs | 23 +- .../src/starfish_rbc_dag_shadow.rs | 328 +++++++++++++++--- docs/starfish-rbc-dag-protocol.md | 21 +- 3 files changed, 304 insertions(+), 68 deletions(-) diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 03dfa7d6..71950faa 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -115,7 +115,6 @@ pub enum CertifiedProjectionError { InvalidLeaderSlot(LeaderSlotV1), MultipleCertifiedLeaderValues(LeaderSlotV1), ConflictingDirectDecision(LeaderSlotV1), - AnchorNotCommitted(ConsensusVertexReference), AnchorTooEarly { slot: LeaderSlotV1, anchor: ConsensusVertexReference, @@ -659,16 +658,22 @@ impl CertifiedProjectionModel { Ok(joined) } - /// Decide an older leader from a later committed anchor. A reachable - /// certifying-round vertex with a QC yields commit; absence yields skip. + /// Decide an older leader from a later projected anchor selected by the + /// ordered committer. A reachable certifying-round vertex with a QC yields + /// commit; absence yields skip. + /// + /// The anchor's frontier is deliberately not required to have been + /// applied yet. The committer first derives the finalized leader sequence + /// from newest to oldest, then applies committed frontiers in the opposite + /// (oldest-to-newest) order. pub fn indirect_decision( &self, slot: LeaderSlotV1, anchor: ConsensusVertexReference, ) -> Result { self.validate_leader_slot(slot)?; - if !self.committed_anchors.contains(&anchor) { - return Err(CertifiedProjectionError::AnchorNotCommitted(anchor)); + if !self.vertices.contains_key(&anchor) { + return Err(CertifiedProjectionError::MissingStrongParent(anchor)); } let minimum_anchor_round = slot.round.saturating_add(3); if anchor.consensus_round() < minimum_anchor_round { @@ -2767,9 +2772,8 @@ mod tests { } #[test] - fn later_committed_anchor_drives_indirect_commit_or_skip() { - let (mut commit_model, slot, leader, commit_anchor) = indirect_graph(true); - commit_model.record_committed_anchor(commit_anchor).unwrap(); + fn later_selected_anchor_drives_indirect_commit_or_skip_before_frontier_application() { + let (commit_model, slot, leader, commit_anchor) = indirect_graph(true); assert_eq!( commit_model.indirect_decision(slot, commit_anchor).unwrap(), ProjectionDecisionV1::IndirectCommit { @@ -2778,8 +2782,7 @@ mod tests { } ); - let (mut skip_model, slot, _, skip_anchor) = indirect_graph(false); - skip_model.record_committed_anchor(skip_anchor).unwrap(); + let (skip_model, slot, _, skip_anchor) = indirect_graph(false); assert_eq!( skip_model.indirect_decision(slot, skip_anchor).unwrap(), ProjectionDecisionV1::IndirectSkip { diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 02699c61..8cb806d7 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -10,7 +10,7 @@ //! only after that complete batch has reached durable storage. use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, VecDeque}, error::Error, fmt, path::Path, @@ -1870,71 +1870,75 @@ impl StarfishRbcDagShadowV1 { fn drive_ordered_committer(&mut self, highest_round: RoundNumber) -> Result<(), ShadowErrorV1> { let decidable_round = highest_round.saturating_sub(2); - loop { - while self.next_undecided_consensus_round <= decidable_round - && self.has_projection_decision( - self.projection - .leader_slot(self.next_undecided_consensus_round), - ) - { - self.next_undecided_consensus_round = - self.next_undecided_consensus_round.saturating_add(1); - } - if self.next_undecided_consensus_round > decidable_round { - return Ok(()); - } - let round = self.next_undecided_consensus_round; + if self.next_undecided_consensus_round > decidable_round { + return Ok(()); + } + + // Plan from newest to oldest, as the universal Starfish committer + // does. An indirect decision may use only the first committed leader + // in this already-decided suffix; a still-undecided intervening slot + // is a hard barrier. This makes anchor selection independent of which + // later direct certificate happened to arrive first locally. + let mut planned = VecDeque::new(); + for round in (self.next_undecided_consensus_round..=decidable_round).rev() { let slot = self.projection.leader_slot(round); - let Ok(decision) = self.projection.direct_decision(slot) else { - return Ok(()); - }; - match decision { - ProjectionDecisionV1::DirectCommit { leader } => { - self.commit_projected_anchor(leader)?; - self.record_projection_decision(decision); - self.next_undecided_consensus_round = round.saturating_add(1); - } - ProjectionDecisionV1::DirectSkip { .. } => { - self.record_projection_decision(decision); - self.next_undecided_consensus_round = round.saturating_add(1); - } + let direct = self.projection.direct_decision(slot)?; + let decision = match direct { ProjectionDecisionV1::Undecided { .. } => { - let first_anchor_round = round.saturating_add(3); - let later_anchor = - (first_anchor_round..=decidable_round).find_map(|candidate| { - let candidate_slot = self.projection.leader_slot(candidate); - match self.projection.direct_decision(candidate_slot).ok()? { - ProjectionDecisionV1::DirectCommit { leader } => Some(leader), - ProjectionDecisionV1::DirectSkip { .. } - | ProjectionDecisionV1::IndirectCommit { .. } - | ProjectionDecisionV1::IndirectSkip { .. } - | ProjectionDecisionV1::Undecided { .. } => None, + let minimum_anchor_round = round.saturating_add(3); + let mut anchor = None; + for later in planned.iter().filter(|later| { + projection_decision_slot(**later).round >= minimum_anchor_round + }) { + match later { + ProjectionDecisionV1::DirectCommit { leader } + | ProjectionDecisionV1::IndirectCommit { leader, .. } => { + anchor = Some(*leader); + break; } - }); - let Some(anchor) = later_anchor else { - return Ok(()); + ProjectionDecisionV1::DirectSkip { .. } + | ProjectionDecisionV1::IndirectSkip { .. } => {} + ProjectionDecisionV1::Undecided { .. } => break, + } + } + let Some(anchor) = anchor else { + planned.push_front(direct); + continue; }; - self.commit_projected_anchor(anchor)?; - let indirect = self - .projection - .indirect_decision(slot, anchor) - .expect("a clean committed later anchor must decide the older slot"); - self.record_projection_decision(indirect); - self.record_projection_decision(ProjectionDecisionV1::DirectCommit { - leader: anchor, - }); - self.next_undecided_consensus_round = round.saturating_add(1); + self.projection.indirect_decision(slot, anchor)? } + ProjectionDecisionV1::DirectCommit { .. } + | ProjectionDecisionV1::DirectSkip { .. } => direct, ProjectionDecisionV1::IndirectCommit { .. } | ProjectionDecisionV1::IndirectSkip { .. } => { unreachable!("direct decision returned an indirect result") } - } + }; + planned.push_front(decision); } - } - fn has_projection_decision(&self, slot: LeaderSlotV1) -> bool { - self.projected_decision_slots.contains(&slot) + // Emit only the longest finalized prefix. Every committed leader, + // including an indirectly committed one, contributes its frontier at + // its exact position in that agreed leader order. Later deciding + // anchors are therefore never applied ahead of older leaders. + for decision in planned { + if matches!(decision, ProjectionDecisionV1::Undecided { .. }) { + break; + } + match decision { + ProjectionDecisionV1::DirectCommit { leader } + | ProjectionDecisionV1::IndirectCommit { leader, .. } => { + self.commit_projected_anchor(leader)?; + } + ProjectionDecisionV1::DirectSkip { .. } + | ProjectionDecisionV1::IndirectSkip { .. } => {} + ProjectionDecisionV1::Undecided { .. } => unreachable!("handled above"), + } + let round = projection_decision_slot(decision).round; + self.record_projection_decision(decision); + self.next_undecided_consensus_round = round.saturating_add(1); + } + Ok(()) } fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { @@ -3536,6 +3540,220 @@ mod tests { } } + fn ordered_committer_vertex( + author: AuthorityIndex, + consensus_round: RoundNumber, + variant: u8, + ) -> ConsensusVertexReference { + let marker = (consensus_round as u8) + .wrapping_mul(17) + .wrapping_add(author as u8) + .wrapping_add(variant.wrapping_mul(71)); + ConsensusVertexReference::new( + BlockReference { + authority: author, + round: 100 + consensus_round * 2 + RoundNumber::from(variant), + digest: BlockDigest::from([marker; 32]), + }, + consensus_round, + ) + } + + fn quorum_authors_including( + committee: &RbcDagCommitteeContextV1, + required: AuthorityIndex, + ) -> Vec { + std::iter::once(required) + .chain( + committee + .committee() + .authorities() + .filter(|author| *author != required), + ) + .take(3) + .collect() + } + + fn inject_ordered_committer_fixture( + node: &mut StarfishRbcDagShadowV1, + include_early_direct_anchor: bool, + ) -> [ConsensusVertexReference; 3] { + let committee = node.committee.clone(); + let leader_author = |round: RoundNumber| committee.committee().elect_leader(round); + let no_vote = |round: RoundNumber| LeaderChoiceV1::NoVote { + leader_author: leader_author(round - 1), + leader_round: round - 1, + }; + let older = ordered_committer_vertex(leader_author(1), 1, 0); + node.projection + .inject_projected_for_test(older, Vec::new(), no_vote(1)); + + // Q voters make the round-one leader certifiable, but only one + // round-three vertex initially carries that certificate. Direct + // commit is therefore unavailable while indirect commit is possible. + let round_two = (0..N as AuthorityIndex) + .map(|author| ordered_committer_vertex(author, 2, 0)) + .collect::>(); + for (author, reference) in round_two.iter().copied().enumerate() { + let choice = if author < 3 { + LeaderChoiceV1::Vote { leader: older } + } else { + no_vote(2) + }; + node.projection + .inject_projected_for_test(reference, vec![older], choice); + } + + let round_three = (0..3 as AuthorityIndex) + .map(|author| ordered_committer_vertex(author, 3, 0)) + .collect::>(); + let round_three_parents = [ + round_two[..3].to_vec(), + vec![round_two[0], round_two[1], round_two[3]], + vec![round_two[1], round_two[2], round_two[3]], + ]; + for (reference, parents) in round_three.iter().copied().zip(round_three_parents) { + node.projection + .inject_projected_for_test(reference, parents, no_vote(3)); + } + + // The first later leader is reachable from the single certificate. + // It is initially only indirectly committed by the still-later + // leader, which is the case the old direct-only anchor scan skipped. + let first_anchor = ordered_committer_vertex(leader_author(4), 4, 0); + let round_four_authors = quorum_authors_including(&committee, first_anchor.author()); + let mut round_four = Vec::new(); + for author in round_four_authors { + let reference = if author == first_anchor.author() { + first_anchor + } else { + ordered_committer_vertex(author, 4, 0) + }; + node.projection + .inject_projected_for_test(reference, round_three.clone(), no_vote(4)); + round_four.push(reference); + } + + let round_five = (0..N as AuthorityIndex) + .map(|author| ordered_committer_vertex(author, 5, 0)) + .collect::>(); + for (author, reference) in round_five.iter().copied().enumerate() { + let choice = if author < 3 { + LeaderChoiceV1::Vote { + leader: first_anchor, + } + } else { + no_vote(5) + }; + node.projection + .inject_projected_for_test(reference, round_four.clone(), choice); + } + + let round_six = (0..3 as AuthorityIndex) + .map(|author| ordered_committer_vertex(author, 6, 0)) + .collect::>(); + let round_six_parents = [ + round_five[..3].to_vec(), + vec![round_five[0], round_five[1], round_five[3]], + vec![round_five[1], round_five[2], round_five[3]], + ]; + for (reference, parents) in round_six.iter().copied().zip(round_six_parents) { + node.projection + .inject_projected_for_test(reference, parents, no_vote(6)); + } + if include_early_direct_anchor { + // One Byzantine author supplies a conflicting certifier while the + // unused fourth author supplies another. Together with author 0, + // direct evidence for the round-four leader reaches Q. + for (author, variant) in [(1, 1), (3, 0)] { + node.projection.inject_projected_for_test( + ordered_committer_vertex(author, 6, variant), + round_five[..3].to_vec(), + no_vote(6), + ); + } + } + + let later_anchor = ordered_committer_vertex(leader_author(7), 7, 0); + let round_seven_authors = quorum_authors_including(&committee, later_anchor.author()); + let mut round_seven = Vec::new(); + for author in round_seven_authors { + let reference = if author == later_anchor.author() { + later_anchor + } else { + ordered_committer_vertex(author, 7, 0) + }; + node.projection + .inject_projected_for_test(reference, round_six.clone(), no_vote(7)); + round_seven.push(reference); + } + let round_eight = (0..3 as AuthorityIndex) + .map(|author| ordered_committer_vertex(author, 8, 0)) + .collect::>(); + for reference in &round_eight { + node.projection.inject_projected_for_test( + *reference, + round_seven.clone(), + LeaderChoiceV1::Vote { + leader: later_anchor, + }, + ); + } + for author in 0..3 as AuthorityIndex { + node.projection.inject_projected_for_test( + ordered_committer_vertex(author, 9, 0), + round_eight.clone(), + no_vote(9), + ); + } + [older, first_anchor, later_anchor] + } + + #[test] + fn ordered_committer_is_deterministic_across_direct_anchor_arrival_orders() { + let mut network = TestNetwork::new(); + let expected = inject_ordered_committer_fixture(&mut network.nodes[0], false); + assert_eq!( + inject_ordered_committer_fixture(&mut network.nodes[1], true), + expected + ); + + network.nodes[0].drive_ordered_committer(9).unwrap(); + network.nodes[1].drive_ordered_committer(9).unwrap(); + + let delayed_decisions = network.nodes[0].drain_projection_decisions(); + let eager_decisions = network.nodes[1].drain_projection_decisions(); + assert!( + delayed_decisions.contains(&ProjectionDecisionV1::IndirectCommit { + leader: expected[1], + anchor: expected[2], + }) + ); + assert!( + eager_decisions.contains(&ProjectionDecisionV1::DirectCommit { + leader: expected[1], + }) + ); + + let delayed_output = network.nodes[0].drain_committed_frontiers(); + let eager_output = network.nodes[1].drain_committed_frontiers(); + assert_eq!(delayed_output, eager_output); + assert_eq!( + delayed_output + .iter() + .map(|delta| delta.anchor) + .collect::>(), + expected + ); + assert_eq!( + delayed_output + .iter() + .map(|delta| delta.output_sequence) + .collect::>(), + vec![1, 2, 3] + ); + } + #[test] fn vote_and_ack_trace_codec_preserves_append_only_golden_tags() { let target = BlockReference { diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 31c78c69..0c67cd8f 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -715,6 +715,13 @@ view is never a no-vote. `NoVote` is explicit, authenticated, immutable, and slo An honest validator persists its leader-choice lock before exposing the carrier that contains it; it cannot emit `NoVote` and later vote for a late leader in the same logical voting slot. +Decision planning follows the existing Starfish newest-to-oldest rule. For an undecided slot, the +deciding anchor is the first committed leader in the already-decided later sequence at least three +rounds ahead. Final skips are traversed, but an intervening undecided slot is a hard barrier; a +validator must not scan past it to whichever later direct certificate happens to be visible first. +Only the longest finalized prefix is published. Direct versus indirect evidence may be observed at +different times, but it cannot change the committed-leader sequence. + Skipping a Byzantine leader role discards only that optional consensus value. It does not discard the enclosing application carrier. If that carrier later becomes part of a closed prefix, a later committed frontier orders its payload. @@ -726,9 +733,9 @@ acknowledgment references; it enters only through the typed Core materialization ## 12. Frontier-delta linearization -Let `F_k` be the effective frontier carried by committed anchor `A_k`, `J_k` the cumulative joined -committed frontier, and `Closure(F)` the union of the exact per-author self-chain prefixes named by -`F`. Maintain: +Let `A_k` be the `k`th committed leader in the finalized leader sequence, whether directly or +indirectly committed, `F_k` its effective frontier, `J_k` the cumulative joined committed frontier, +and `Closure(F)` the union of the exact per-author self-chain prefixes named by `F`. Maintain: ```text J_0 = [None; committee_size] @@ -742,6 +749,12 @@ component already committed. Before outputting `Delta`, a validator requires eve to be authoritatively delivered and every application member to pass the concrete Core DA gate. Exact carrier recovery and the dedicated verified-payload path supply missing material. +Frontiers are applied strictly in increasing finalized-leader order. A later leader used to decide +an older slot is planning evidence only until every older slot has resolved; its frontier is not +applied ahead of an indirectly committed older leader. Thus different projection arrival orders +may classify a leader as direct or indirect at different times, but produce the same ordered anchor +and frontier-delta sequence. + All validators deterministically order the same delta by `(carrier_round, author, content_digest)`. Because a closed author prefix advances by exactly one carrier round, this key already preserves mandatory `own_prev` order. @@ -945,6 +958,8 @@ The executable model and composed runtime tests should cover at minimum: - an authoritatively delivered Byzantine carrier above an unavailable self-chain gap; - conflicting Byzantine consensus vertices in one logical slot; - explicit vote/no-vote conflicts and direct plus indirect commit/skip; +- different projection arrival orders where a later direct certificate precedes an earlier + indirect anchor, with byte-identical committed-leader and cumulative frontier-delta sequences; - frontier fork, regression, and strong-parent dominance rejection; - concurrent committed anchors producing the same cumulative joined frontier and byte-identical output deltas without regression; From 9cdbc16776a90199ed8f6a41a3f169a24c9f4cbb Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:23:34 +0200 Subject: [PATCH 38/62] Instrument RBC-DAG consensus cadence --- crates/starfish-core/src/config.rs | 15 ++ crates/starfish-core/src/metrics.rs | 11 ++ crates/starfish-core/src/net_sync.rs | 3 + .../src/starfish_rbc_dag_shadow.rs | 14 ++ .../src/starfish_rbc_dag_shadow_service.rs | 151 +++++++++++++++++- crates/starfish-core/src/validator.rs | 51 ++++++ crates/starfish/src/main.rs | 18 +++ docs/starfish-rbc-dag-protocol.md | 14 +- 8 files changed, 270 insertions(+), 7 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index d1dd250a..6c90f2c4 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -76,6 +76,13 @@ pub struct NodeParameters { /// This remains experimental and requires `starfish_rbc_dag_shadow`. #[serde(default)] pub starfish_rbc_dag_autonomous_clock: bool, + /// Optional logical C2 fallback timeout for the autonomous RBC-DAG. + /// `None` preserves the historical behavior and reuses `leader_timeout`. + /// This is deliberately independent of the physical carrier heartbeat so + /// latency experiments can vary consensus fallback without changing the + /// proactive push cadence. + #[serde(default)] + pub starfish_rbc_dag_consensus_timeout: Option, /// Use embedded carrier ECHO/READY delivery as the certification authority /// for Starfish-RBC application headers. Direct RBC retains INIT/payload /// transport but its phase messages cannot mark a block clean. @@ -164,6 +171,7 @@ impl Default for NodeParameters { starfish_rbc_protocol_instance: None, starfish_rbc_dag_shadow: false, starfish_rbc_dag_autonomous_clock: false, + starfish_rbc_dag_consensus_timeout: None, starfish_rbc_dag_embedded_rbc_authority: false, starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), @@ -438,6 +446,7 @@ mod tests { assert_eq!(parameters.starfish_rbc_protocol_instance, None); assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); + assert_eq!(parameters.starfish_rbc_dag_consensus_timeout, None); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); @@ -451,6 +460,7 @@ mod tests { ); assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); + assert_eq!(decoded.starfish_rbc_dag_consensus_timeout, None); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); } @@ -460,6 +470,7 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, leader_timeout: Duration::from_millis(125), + starfish_rbc_dag_consensus_timeout: Some(Duration::from_millis(75)), starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -470,6 +481,10 @@ mod tests { assert!(decoded.starfish_rbc_dag_autonomous_clock); assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(decoded.leader_timeout, Duration::from_millis(125)); + assert_eq!( + decoded.starfish_rbc_dag_consensus_timeout, + Some(Duration::from_millis(75)) + ); } #[test] diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index f118bc4a..916c6fc7 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -2431,6 +2431,17 @@ impl Metrics { .unwrap_or_default(), ) ]); + table.add_row(row![ + b->"Logical vertex creation:", + format!( + "bootstrap/C1/C2/C3/omitted={}/{}/{}/{}/{}", + shadow_input_count("consensus_vertex", "bootstrap"), + shadow_input_count("consensus_vertex", "c1"), + shadow_input_count("consensus_vertex", "c2"), + shadow_input_count("consensus_vertex", "c3"), + shadow_input_count("consensus_vertex", "omitted"), + ) + ]); let stage_latency = RBC_DAG_PIPELINE_LATENCY_STAGES .iter() .map(|stage| { diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index e2789e11..05d801f9 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -3222,6 +3222,9 @@ impl NetworkSyncer // resolved Starfish leader timeout. Application and // embedded RBC phase carriers remain event-driven. node_parameters.leader_timeout, + node_parameters + .starfish_rbc_dag_consensus_timeout + .unwrap_or(node_parameters.leader_timeout), wal_sync_policy, Arc::clone(&metrics), rbc_dag_frontier_recovery_cursor, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 8cb806d7..c7f9f082 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -1340,6 +1340,20 @@ impl StarfishRbcDagShadowV1 { self.next_local_consensus_round } + /// Whether the currently open logical slot can be fixed through the + /// optimistic C1 path using only already-promised projection state. This + /// is a pure preview of the same builder used for durable carrier + /// creation; callers must recompute when they actually create. + pub(crate) fn local_consensus_vertex_c1_ready(&self) -> bool { + if self.next_local_consensus_round <= 1 { + return false; + } + let Ok((own_prev, _)) = self.model.local_parent_set() else { + return false; + }; + self.build_local_consensus_vertex(own_prev, false).is_some() + } + pub(crate) fn projected_consensus_stake(&self, round: RoundNumber) -> Stake { self.promised_projection.projected_stake_at_round(round) } diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index d4504394..c23c4501 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -1004,6 +1004,7 @@ pub(crate) enum ShadowServiceErrorV1 { reference: BlockReference, }, InvalidHeartbeatInterval, + InvalidConsensusTimeout, SyncRequestForForeignAuthor { expected: AuthorityIndex, actual: AuthorityIndex, @@ -1115,6 +1116,9 @@ impl fmt::Display for ShadowServiceErrorV1 { Self::InvalidHeartbeatInterval => formatter.write_str( "Starfish-RBC-DAG autonomous heartbeat interval must be nonzero", ), + Self::InvalidConsensusTimeout => formatter.write_str( + "Starfish-RBC-DAG logical consensus timeout must be nonzero", + ), Self::SyncRequestForForeignAuthor { expected, actual } => write!( formatter, "shadow carrier sync request asked authority {expected} to serve authority {actual}" @@ -1177,6 +1181,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::DirectMirror, + None, wal_sync_policy, None, None, @@ -1210,6 +1215,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::DirectMirror, + None, wal_sync_policy, Some(metrics), None, @@ -1246,6 +1252,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + None, wal_sync_policy, None, None, @@ -1283,6 +1290,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + None, wal_sync_policy, Some(metrics), None, @@ -1324,6 +1332,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_paused_with_metric authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + None, wal_sync_policy, Some(metrics), None, @@ -1344,6 +1353,7 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, heartbeat_interval: Duration, + consensus_timeout: Duration, wal_sync_policy: ShadowWalSyncPolicyV1, metrics: Arc, recovery_cursor: Option, @@ -1359,6 +1369,9 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 if heartbeat_interval.is_zero() { return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); } + if consensus_timeout.is_zero() { + return Err(ShadowServiceErrorV1::InvalidConsensusTimeout); + } start_starfish_rbc_dag_shadow_service_with_mode_v1( path, committee, @@ -1367,6 +1380,7 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + Some(consensus_timeout), wal_sync_policy, Some(metrics), recovery_cursor, @@ -1448,6 +1462,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, mode: ShadowServiceModeV1, + consensus_timeout: Option, wal_sync_policy: ShadowWalSyncPolicyV1, metrics: Option>, recovery_cursor: Option, @@ -1461,6 +1476,9 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( ShadowServiceErrorV1, > { let committee_size = committee.committee().len(); + let consensus_timeout = consensus_timeout + .or_else(|| mode.heartbeat_interval()) + .unwrap_or_default(); let input_capacity = shadow_input_capacity(committee_size, mode)?; let max_sidecar_size = authentication_sidecar_size(context.authentication_scheme(), committee_size); @@ -1903,6 +1921,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( sync_max_desired_responses: 0, awaiting_application_submission: false, consensus_pacemaker, + consensus_timeout, normal_carrier_min_spacing: mode .heartbeat_interval() .and_then(|interval| interval.checked_div(SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1)) @@ -2176,6 +2195,10 @@ struct ShadowServiceStateV1 { /// control heartbeat. Maintenance/heartbeat messages bound the wait. awaiting_application_submission: bool, consensus_pacemaker: ConsensusPacemakerV1, + /// Logical C2 fallback deadline. This is independent of the physical + /// heartbeat so experiments can vary consensus permission without + /// changing proactive carrier push cadence. + consensus_timeout: Duration, normal_carrier_min_spacing: Duration, normal_carrier_next_allowed_at: Option, normal_carrier_requested: bool, @@ -3081,9 +3104,10 @@ impl ShadowServiceStateV1 { if !self.clock_active { return false; } - let Some(leader_timeout) = self.mode.heartbeat_interval() else { + if !self.mode.is_autonomous() { return false; - }; + } + let leader_timeout = self.consensus_timeout; let slot = self.core.next_local_consensus_round(); let a1_ready = slot == 1 || self @@ -3268,6 +3292,9 @@ impl ShadowServiceStateV1 { .try_into() .unwrap_or(TimestampNs::MAX); let before = self.core.wal_counts(); + let consensus_slot = self.core.next_local_consensus_round(); + let c1_ready = self.core.local_consensus_vertex_c1_ready(); + let c3_ready = self.core.has_projected_consensus_quorum(consensus_slot); let application_round = self.pending_local.keys().next().copied(); let application = application_round.and_then(|round| self.pending_local.remove(&round)); let result = match &application { @@ -3282,6 +3309,8 @@ impl ShadowServiceStateV1 { }; match result { Ok((envelope, effects)) => { + let fixed_consensus_vertex = + self.core.next_local_consensus_round() > consensus_slot; self.record_carrier_created(Instant::now()); let initial_application_payload = application .as_ref() @@ -3317,6 +3346,22 @@ impl ShadowServiceStateV1 { }, outcome: "accepted", }); + self.emit(ShadowServiceEventV1::Input { + kind: "consensus_vertex", + outcome: if !fixed_consensus_vertex { + "omitted" + } else if consensus_slot == 1 { + "bootstrap" + } else if c1_ready { + "c1" + } else if c3_ready { + "c3" + } else if allow_no_vote { + "c2" + } else { + "unexpected" + }, + }); self.report_wal_delta(before); if let Some(reference) = assigned_application { self.awaiting_application_submission = true; @@ -4227,6 +4272,12 @@ fn run_shadow_service( } state.report_wal_delta(before); state.process_effects(outcome.effects().to_vec()); + // A coalesced producer notification may sit behind + // this ingress in the bounded actor FIFO even though + // its exact application is already present in the + // shared desired map. Reconcile it before phase work + // consumes the carrier round that this ingress opens. + state.reconcile_local_applications(); state.retry_pending_local(); if future_ignored { state.flush_carrier_sync_requests(true); @@ -5218,11 +5269,14 @@ mod tests { ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), ) .unwrap(); - let leader_timeout = Duration::from_millis(20); + let heartbeat_interval = Duration::from_secs(60); + let consensus_timeout = Duration::from_millis(20); let (mut state, mut event_rx, mut message_rx, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), leader_timeout); + standalone_autonomous_state(core, harness.committee.clone(), heartbeat_interval); + state.consensus_timeout = consensus_timeout; state.refresh_consensus_pacemaker(); + assert_eq!(state.mode.heartbeat_interval(), Some(heartbeat_interval)); assert_eq!(state.core.local_carrier_round(), 1); let (generation, slot) = match timeout(Duration::from_secs(1), message_rx.recv()) .await @@ -5550,6 +5604,7 @@ mod tests { ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), Vec::new(), ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, + None, ShadowWalSyncPolicyV1::EveryBatch, None, None, @@ -5646,6 +5701,7 @@ mod tests { sync_max_desired_responses: 0, awaiting_application_submission: false, consensus_pacemaker: ConsensusPacemakerV1::new(slot), + consensus_timeout: leader_timeout, // Existing state-level tests invoke creation synchronously and do // not run the service deadline task. Keep their historical // immediate behavior unless a pacer test overrides this field. @@ -8253,6 +8309,93 @@ mod tests { stop(handle, events, task).await; } + #[tokio::test] + async fn coalesced_application_wins_the_round_opened_by_carrier_ingress() { + let harness = Harness::new(); + let (handle, mut events, task) = + harness.start_autonomous_with_interval(0, Duration::from_millis(600)); + wait_ready(&mut events).await; + let (carrier_tx, mut carrier_rx) = mpsc::unbounded_channel(); + let event_task = tokio::spawn(async move { + while let Some(event) = events.recv().await { + match event { + ShadowServiceEventV1::Network { + recipient: 1, + message: NetworkMessage::RbcDagShadowCarrier(envelope), + } => carrier_tx.send(envelope).unwrap(), + ShadowServiceEventV1::Rejected { error, .. } => { + panic!("coalesced application test rejected input: {error}") + } + _ => {} + } + } + }); + + handle.local_header(&direct_header(0, 1, 0x71)).unwrap(); + let first = timeout(EVENT_TIMEOUT, carrier_rx.recv()) + .await + .expect("first local carrier timed out") + .expect("carrier collector stopped"); + let first = CandidateCarrierV1::decode_wire_with_committee( + &first.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + assert_eq!(first.header().carrier_round(), 1); + tokio::time::sleep(Duration::from_millis(40)).await; + + let first_peer = round_one_candidate(1, &harness.committee, 0x72); + handle.carrier(1, harness.envelope(&first_peer, 1)).unwrap(); + let inspection = handle.inspect_carrier_sync().await.unwrap(); + assert_eq!(inspection.open_round, 1); + + // Model a full/coalesced notification queue: the producer has + // published the exact desired application, but its wake is not ahead + // of the final quorum carrier in the actor FIFO. + let application = RbcCanonicalHeader::try_new( + 0, + 2, + (0..N as AuthorityIndex) + .map(|author| BlockReference::new_test(author, 1)) + .collect(), + Vec::new(), + 2_073, + TransactionsCommitment::from_bytes([0x73; 32]), + ) + .unwrap(); + handle.desired_local_applications.lock().insert( + application.reference().round, + ShadowLocalCarrierV1::from_direct_header(&application), + ); + let quorum_peer = round_one_candidate(2, &harness.committee, 0x74); + handle + .carrier(2, harness.envelope(&quorum_peer, 2)) + .unwrap(); + let inspection = handle.inspect_carrier_sync().await.unwrap(); + assert_eq!(inspection.open_round, 2); + + let second = loop { + let envelope = timeout(EVENT_TIMEOUT, carrier_rx.recv()) + .await + .expect("second local carrier timed out") + .expect("carrier collector stopped"); + let candidate = CandidateCarrierV1::decode_wire_with_committee( + &envelope.canonical_carrier, + &harness.committee, + None, + ) + .unwrap(); + if candidate.header().carrier_round() == 2 { + break candidate; + } + }; + assert_eq!(second.header().application_header(), Some(&application)); + handle.shutdown().await.unwrap(); + task.await.unwrap(); + event_task.await.unwrap(); + } + #[tokio::test] async fn recovery_binds_holder_and_reference_then_compares_only_paired_slot_once() { let harness = Harness::new(); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 1fe59af4..77518c4e 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -95,6 +95,25 @@ impl Validator { "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" )); } + if public_config + .parameters + .starfish_rbc_dag_consensus_timeout + .is_some_and(|timeout| timeout.is_zero()) + { + return Err(eyre!( + "Starfish-RBC-DAG logical consensus timeout must be nonzero" + )); + } + if public_config + .parameters + .starfish_rbc_dag_consensus_timeout + .is_some() + && !public_config.parameters.starfish_rbc_dag_autonomous_clock + { + return Err(eyre!( + "Starfish-RBC-DAG logical consensus timeout requires the autonomous clock" + )); + } if start_options.rbc_dag_clock_start_paused && (!public_config.parameters.starfish_rbc_dag_autonomous_clock || !public_config @@ -504,6 +523,38 @@ mod smoke_tests { })); } + #[tokio::test] + async fn logical_consensus_timeout_requires_autonomous_clock() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_consensus_timeout = + Some(Duration::from_millis(250)); + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("logical consensus timeout requires the autonomous clock") + })); + } + #[tokio::test] async fn buffered_shadow_wal_requires_shadow_mode() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 5bff7ddd..d1f9dc88 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -205,6 +205,10 @@ enum Operation { /// headers. Requires the autonomous RBC-DAG mode. #[clap(long, default_value_t = false)] starfish_rbc_dag_embedded_rbc_authority: bool, + /// Override only the autonomous RBC-DAG logical C2 fallback timeout. + /// The physical carrier heartbeat remains on `leader_timeout`. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_consensus_timeout_ms: Option, /// Benchmark-only: write ordered shadow-WAL frames but force them to /// stable storage only at clean shutdown. This run is not crash-safe. #[clap(long, default_value_t = false)] @@ -327,6 +331,7 @@ async fn main() -> Result<()> { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, duration_secs, port_offset, @@ -343,6 +348,8 @@ async fn main() -> Result<()> { node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; node_parameters.starfish_rbc_dag_embedded_rbc_authority = starfish_rbc_dag_embedded_rbc_authority; + node_parameters.starfish_rbc_dag_consensus_timeout = + starfish_rbc_dag_consensus_timeout_ms.map(Duration::from_millis); node_parameters.starfish_rbc_dag_shadow_buffered_wal = starfish_rbc_dag_shadow_buffered_wal; if consensus_protocol == "starfish-rbc" { @@ -576,6 +583,13 @@ async fn local_benchmark( "Carrier idle timeout: {} ms (shared Starfish leader pacemaker)", node_parameters.leader_timeout.as_millis() ); + println!( + "Logical C2 timeout: {} ms", + node_parameters + .starfish_rbc_dag_consensus_timeout + .unwrap_or(node_parameters.leader_timeout) + .as_millis() + ); } if let Some(latency) = node_parameters.uniform_latency_ms { println!("Network Latency: {latency} ms (uniform)"); @@ -1508,6 +1522,8 @@ mod tests { "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-embedded-rbc-authority", + "--starfish-rbc-dag-consensus-timeout-ms", + "250", "--starfish-rbc-dag-shadow-buffered-wal", "--port-offset", "2500", @@ -1520,6 +1536,7 @@ mod tests { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, port_offset, .. @@ -1532,6 +1549,7 @@ mod tests { assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_dag_embedded_rbc_authority); + assert_eq!(starfish_rbc_dag_consensus_timeout_ms, Some(250)); assert!(starfish_rbc_dag_shadow_buffered_wal); assert_eq!(port_offset, 2500); } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 0c67cd8f..98cccb09 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -49,6 +49,9 @@ heartbeats use the resolved Starfish leader timeout (600 ms for Push/Starfish-RB application carriers and encodable phase follow-ups are event-driven. The V4 authoritative mode promotes the proved `O`-ECHO predicate to delivery authority; `Q` READY remains a distinct slower certification fact rather than a prerequisite for fast projection. +The research harness may override the logical C2 fallback timeout independently for controlled +latency experiments. This does not change the physical heartbeat, carrier pacing, C1/C3 rules, or +wire format; absent an override, C2 continues to use the resolved Starfish leader timeout. Standalone authority is an end-to-end boundary. The direct Starfish-RBC actor is absent, generic block dissemination and legacy pull messages are rejected or ignored, and the legacy Starfish @@ -611,7 +614,8 @@ an authoritative optimistic delivery: - **C1:** create at `c` after the eligible leader at `c - 1` is present and the eligible projection contains either `Q` votes for an exact leader value or a valid explicit direct-skip pattern for the leader slot at `c - 2`; -- **C2:** create after the consensus leader timeout; or +- **C2:** create after the logical consensus timeout, which defaults to the resolved Starfish + leader timeout; or - **C3:** catch up and create after observing eligible distinct-author stake `Q` already at `c`. The strong-parent set chosen under C1 must itself contain the immutable L2 witness: the exact `Q` @@ -952,6 +956,9 @@ The executable model and composed runtime tests should cover at minimum: independent `Q`-READY certification; - evidence-before-content recovery from phase holders, including VOTE/ACK without local admission; - zero application load with heartbeat-only RBC completion; +- independent logical-C2 timeout scheduling without changing the physical heartbeat, plus + coalesced producer notification ordering in which an already-published application wins a newly + opened carrier round before queued phase-only work; - two-round admission, 64-round authenticated retention, and future carriers that cannot jump the local sequential clock; - `f` permanently missing weak parents without blocking honest carrier or consensus progress; @@ -1091,8 +1098,9 @@ The following production choices remain unresolved and must be proved or measure admission lookahead `2`, retains at most `64` future rounds for temporarily descheduled peers, and discards farther unsolicited carriers before admission/retention; these are benchmark resource parameters rather than protocol safety constants); -- whether the shared Starfish leader-timeout policy needs a separately proved adaptive low-load - rule; the prototype intentionally does not introduce a second heartbeat timeout; +- whether the logical C2 timeout needs a separately proved adaptive low-load rule; the prototype + exposes an experimental override but intentionally does not introduce a second physical + heartbeat timeout; - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and From 7ef98f20a205b65a8b1a1b97ca197ec8e60b8ae2 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:51:41 +0200 Subject: [PATCH 39/62] Authenticate RBC-DAG carrier recovery relays --- crates/starfish-core/src/net_sync.rs | 24 ++ crates/starfish-core/src/network.rs | 31 ++ .../src/starfish_rbc_dag_shadow.rs | 140 +++++++++ .../src/starfish_rbc_dag_shadow_service.rs | 275 +++++++++++++++++- docs/starfish-rbc-dag-protocol.md | 37 ++- 5 files changed, 490 insertions(+), 17 deletions(-) diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 05d801f9..f78cb841 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -109,6 +109,7 @@ enum RbcDagOutboundKeyV1 { Proactive(BlockReference), CarrierRequest(BlockReference), CarrierResponse(BlockReference), + CarrierEnvelopeResponse(BlockReference), SyncRequest(AuthorityIndex, RoundNumber), SyncResponse(AuthorityIndex, RoundNumber), ApplicationPayloadRequest(BlockReference), @@ -481,6 +482,10 @@ fn rbc_dag_outbound_classification( priority, RbcDagOutboundKeyV1::CarrierResponse(response.reference), )), + NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(response) => Ok(( + priority, + RbcDagOutboundKeyV1::CarrierEnvelopeResponse(response.reference), + )), NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => Ok(( priority, RbcDagOutboundKeyV1::SyncRequest(request.author, request.round), @@ -514,6 +519,10 @@ fn rbc_dag_outbound_messages_equal(left: &NetworkMessage, right: &NetworkMessage NetworkMessage::RbcDagShadowCarrierResponse(left), NetworkMessage::RbcDagShadowCarrierResponse(right), ) => left == right, + ( + NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(left), + NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(right), + ) => left == right, ( NetworkMessage::RbcDagShadowCarrierSyncRequest(left), NetworkMessage::RbcDagShadowCarrierSyncRequest(right), @@ -1738,6 +1747,21 @@ impl ConnectionHandler { + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow + .carrier_envelope_response_reliably(self.peer_id, response) + .await + { + if shadow_transport_error_invalidates_run(&error) { + invalidate_shadow_run(&self.metrics); + } + tracing::warn!( + "Failed to forward RBC-DAG shadow envelope response: {error}" + ); + } + } + } NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 86856ae5..63799242 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -123,6 +123,18 @@ pub struct RbcDagShadowCarrierResponse { pub canonical_carrier: Vec, } +/// Canonical carrier content plus one exact authentication-sidecar variant +/// retained by a phase-evidence holder. The requester recomputes `reference` +/// and verifies only its receiver-specific entry. A valid entry grants the +/// same authority as ordinary relayed ingress; an invalid entry falls back to +/// content-only recovery without blaming the author or holder. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct RbcDagShadowCarrierEnvelopeResponse { + pub reference: BlockReference, + pub canonical_carrier: Vec, + pub authentication_sidecar: Vec, +} + /// Request one exact carrier-clock slot from a peer. Keeping synchronization /// slot-addressed prevents an untrusted peer from choosing an unbounded range /// of history to return. @@ -279,6 +291,10 @@ pub enum NetworkMessage { /// Return commitment-checked transaction data for an authorized embedded /// application header. The response is not an author proof. RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse), + /// Return canonical carrier content with one exact retained + /// authentication-sidecar variant. Appended after the frozen V1 message + /// family so every preceding bincode enum discriminant remains stable. + RbcDagShadowCarrierEnvelopeResponse(RbcDagShadowCarrierEnvelopeResponse), } impl NetworkMessage { @@ -313,6 +329,9 @@ impl NetworkMessage { Self::RbcDagShadowCarrierSyncResponse(_) => "rbc_dag_shadow_carrier_sync_response", Self::RbcDagApplicationPayloadRequest(_) => "rbc_dag_application_payload_request", Self::RbcDagApplicationPayloadResponse(_) => "rbc_dag_application_payload_response", + Self::RbcDagShadowCarrierEnvelopeResponse(_) => { + "rbc_dag_shadow_carrier_envelope_response" + } } } } @@ -1944,6 +1963,13 @@ mod tests { reference: block_ref, canonical_carrier: vec![0xA6, 0xA7], }); + let shadow_envelope_response = NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( + RbcDagShadowCarrierEnvelopeResponse { + reference: block_ref, + canonical_carrier: vec![0xB6, 0xB7], + authentication_sidecar: vec![0xB8, 0xB9], + }, + ); let sync_request = NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { author: 2, @@ -1975,6 +2001,11 @@ mod tests { (sync_response, 19, "rbc_dag_shadow_carrier_sync_response"), (payload_request, 20, "rbc_dag_application_payload_request"), (payload_response, 21, "rbc_dag_application_payload_response"), + ( + shadow_envelope_response, + 22, + "rbc_dag_shadow_carrier_envelope_response", + ), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index c7f9f082..0e7e5737 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -1618,6 +1618,55 @@ impl StarfishRbcDagShadowV1 { self.apply_requested_recovery(candidate) } + /// Recover an exact phase-evidenced carrier through the ordinary + /// receiver-bound authentication predicate. A valid sidecar variant is + /// durably recorded as relayed/direct authenticated ingress; an invalid + /// variant retains the exact requested content through the existing + /// content-only recovery path. MAC failures assign no blame. + pub(crate) fn recover_or_admit_from_peer( + &mut self, + expected_reference: BlockReference, + canonical_carrier_wire: &[u8], + authentication_sidecar: &[u8], + trusted_peer: AuthorityIndex, + ) -> Result { + self.ensure_live()?; + if !self.committee.committee().known_authority(trusted_peer) { + return Err(ShadowErrorV1::UnknownAuthority(trusted_peer)); + } + if !self.requested_recoveries.contains_key(&expected_reference) { + return Err(ShadowErrorV1::UnrequestedRecovery(expected_reference)); + } + let candidate = decode_candidate( + canonical_carrier_wire, + &self.committee, + Some(expected_reference), + )?; + let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); + validate_provenance(provenance, candidate.header().author(), &self.committee)?; + + match self.authenticate_decoded(candidate.clone(), authentication_sidecar) { + Ok(authenticated) => { + let (effects, applied) = + self.apply_authenticated_capability(authenticated, provenance)?; + if applied { + return Ok(ShadowIngressOutcomeV1::new( + ShadowIngressDispositionV1::Authenticated, + effects, + )); + } + } + Err(ShadowErrorV1::Carrier(_)) | Err(ShadowErrorV1::NonCanonicalAuthentication) => {} + Err(error) => return Err(error), + } + + let effects = self.apply_requested_recovery(candidate)?; + Ok(ShadowIngressOutcomeV1::new( + ShadowIngressDispositionV1::CandidateRetained, + effects, + )) + } + pub(crate) fn retained_candidate_wire(&self, reference: BlockReference) -> Option> { self.journal .snapshot() @@ -1625,6 +1674,31 @@ impl StarfishRbcDagShadowV1 { .map(<[u8]>::to_vec) } + /// Return one exact, durably retained authentication variant for a + /// carrier. Locally authored envelopes use the outbound WAL record; + /// received envelopes use the first authenticated ingress record, whose + /// provenance and bytes replay identically after restart. + pub(crate) fn retained_authenticated_envelope( + &self, + reference: BlockReference, + ) -> Option { + if reference.authority == self.own_authority { + return self + .local_outbound_envelope(reference.round) + .filter(|envelope| envelope.reference() == reference); + } + self.journal + .snapshot() + .authenticated_ingress() + .iter() + .find(|ingress| ingress.reference() == reference) + .map(|ingress| ShadowOutboundEnvelopeV1 { + reference, + canonical_carrier_wire: ingress.canonical_carrier_wire().to_vec(), + authentication_sidecar: ingress.authentication_sidecar().to_vec(), + }) + } + /// Decode canonical carrier bytes without mutating the reducer. Sync /// clients use this to bind a response to the requested author and round /// before passing it through normal authenticated ingress. @@ -4938,6 +5012,72 @@ mod tests { assert_eq!(network.nodes[0].wal_counts(), (0, 0)); } + #[test] + fn relayed_authenticated_envelope_reopens_with_exact_provenance_and_bytes() { + let mut network = TestNetwork::new(); + let candidate = round_one_candidate(1, &network.committee, 0x80); + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: 1, + keys: &network.keyrings[1], + }, + ) + .unwrap(); + let wire = candidate.canonical_wire_bytes().unwrap(); + let sidecar = authentication.canonical_wire_bytes(); + network.nodes[0] + .receive_authenticated_from_peer(&wire, &sidecar, 2) + .unwrap(); + assert_eq!( + network.nodes[0].admitted_reference(1, 1), + Some(candidate.reference()) + ); + assert_eq!( + network.nodes[0].journal.snapshot().authenticated_ingress()[0].provenance(), + IngressProvenanceV1::Relayed { peer: 2 } + ); + let retained = network.nodes[0] + .retained_authenticated_envelope(candidate.reference()) + .unwrap(); + assert_eq!(retained.canonical_carrier_wire(), wire); + assert_eq!(retained.authentication_sidecar(), sidecar); + + let node = network.nodes.swap_remove(0); + let path = network.path(0); + node.shutdown().unwrap(); + let (reopened, report) = StarfishRbcDagShadowV1::open( + path, + network.committee.clone(), + 0, + network.context, + ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), + ) + .unwrap(); + assert_eq!(report.replayed_batches(), 1); + assert_eq!( + reopened.authenticated_reference(1, 1), + Some(candidate.reference()) + ); + assert_eq!( + reopened.admitted_reference(1, 1), + Some(candidate.reference()) + ); + assert_eq!( + reopened.journal.snapshot().authenticated_ingress()[0].provenance(), + IngressProvenanceV1::Relayed { peer: 2 } + ); + let retained = reopened + .retained_authenticated_envelope(candidate.reference()) + .unwrap(); + assert_eq!(retained.canonical_carrier_wire(), wire); + assert_eq!(retained.authentication_sidecar(), sidecar); + reopened.shutdown().unwrap(); + } + #[test] fn replay_rejects_a_duplicate_authenticated_slot_even_with_an_exact_trace() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index c23c4501..0b9842c9 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -36,8 +36,8 @@ use crate::{ }, network::{ NetworkMessage, RbcDagApplicationPayloadResponse, RbcDagShadowCarrier, - RbcDagShadowCarrierResponse, RbcDagShadowCarrierSyncRequest, - RbcDagShadowCarrierSyncResponse, + RbcDagShadowCarrierEnvelopeResponse, RbcDagShadowCarrierResponse, + RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ @@ -310,6 +310,10 @@ enum ShadowServiceMessageV1 { peer: AuthorityIndex, response: RbcDagShadowCarrierResponse, }, + CarrierEnvelopeResponse { + peer: AuthorityIndex, + response: RbcDagShadowCarrierEnvelopeResponse, + }, CarrierSyncRequest { peer: AuthorityIndex, request: RbcDagShadowCarrierSyncRequest, @@ -540,6 +544,42 @@ impl StarfishRbcDagShadowServiceHandleV1 { .await } + #[cfg(test)] + pub(crate) fn carrier_envelope_response( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierEnvelopeResponse, + ) -> Result<(), ShadowServiceErrorV1> { + self.validate_carrier_envelope_response(&response)?; + self.send(ShadowServiceMessageV1::CarrierEnvelopeResponse { peer, response }) + } + + pub(crate) async fn carrier_envelope_response_reliably( + &self, + peer: AuthorityIndex, + response: RbcDagShadowCarrierEnvelopeResponse, + ) -> Result<(), ShadowServiceErrorV1> { + self.validate_carrier_envelope_response(&response)?; + self.send_reliably(ShadowServiceMessageV1::CarrierEnvelopeResponse { peer, response }) + .await + } + + fn validate_carrier_envelope_response( + &self, + response: &RbcDagShadowCarrierEnvelopeResponse, + ) -> Result<(), ShadowServiceErrorV1> { + validate_wire_size( + "carrier envelope response", + response.canonical_carrier.len(), + MAX_CARRIER_CONTENT_SIZE_V1, + )?; + validate_wire_size( + "carrier envelope response authentication sidecar", + response.authentication_sidecar.len(), + self.max_sidecar_size, + ) + } + #[cfg(test)] pub(crate) fn carrier_sync_request( &self, @@ -854,6 +894,7 @@ impl ShadowServiceMessageV1 { Self::Carrier { .. } => "carrier", Self::CarrierRequest { .. } => "carrier_request", Self::CarrierResponse { .. } => "carrier_response", + Self::CarrierEnvelopeResponse { .. } => "carrier_envelope_response", Self::CarrierSyncRequest { .. } => "carrier_sync_request", Self::CarrierSyncResponsesChanged => "carrier_sync_responses_changed", Self::ApplicationPayloadRequest { .. } => "application_payload_request", @@ -4307,7 +4348,20 @@ fn run_shadow_service( state.reject(Some(peer), error); continue; } - if let Some(canonical_carrier) = state.core.retained_candidate_wire(reference) { + if let Some(envelope) = state.core.retained_authenticated_envelope(reference) { + state.emit(ShadowServiceEventV1::Network { + recipient: peer, + message: NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( + RbcDagShadowCarrierEnvelopeResponse { + reference, + canonical_carrier: envelope.canonical_carrier_wire().to_vec(), + authentication_sidecar: envelope.authentication_sidecar().to_vec(), + }, + ), + }); + } else if let Some(canonical_carrier) = + state.core.retained_candidate_wire(reference) + { state.emit(ShadowServiceEventV1::Network { recipient: peer, message: NetworkMessage::RbcDagShadowCarrierResponse( @@ -4391,6 +4445,93 @@ fn run_shadow_service( } } } + ShadowServiceMessageV1::CarrierEnvelopeResponse { peer, response } => { + if let Err(error) = state.validate_peer(peer) { + state.reject(Some(peer), error); + continue; + } + if state + .core + .authenticated_reference(response.reference.authority, response.reference.round) + == Some(response.reference) + { + state.emit(ShadowServiceEventV1::Input { + kind: "recovery", + outcome: "ignored_already_authenticated", + }); + continue; + } + let Some(holders) = state.pending_recovery.get(&response.reference) else { + state.reject( + Some(peer), + ShadowServiceErrorV1::UnexpectedResponse(response.reference), + ); + continue; + }; + if !holders.contains(&peer) { + state.reject( + Some(peer), + ShadowServiceErrorV1::ResponseFromNonHolder { + peer, + reference: response.reference, + }, + ); + continue; + } + let before = state.core.wal_counts(); + match state.core.recover_or_admit_from_peer( + response.reference, + &response.canonical_carrier, + &response.authentication_sidecar, + peer, + ) { + Ok(outcome) => { + state.pending_recovery.remove(&response.reference); + state + .recovery_last_attempt + .retain(|(target, _), _| *target != response.reference); + state.emit(ShadowServiceEventV1::Input { + kind: "recovery", + outcome: match outcome.disposition() { + ShadowIngressDispositionV1::Authenticated => { + "accepted_authenticated" + } + ShadowIngressDispositionV1::CandidateRetained => { + "accepted_content_only" + } + ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { + "ignored_duplicate" + } + ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => { + "future_ignored" + } + }, + }); + if let Err(error) = state.observe_carrier_application( + peer, + &response.canonical_carrier, + None, + outcome.disposition(), + ) { + state.reject(Some(peer), error); + } + state.report_wal_delta(before); + state.process_effects(outcome.effects().to_vec()); + state.retry_pending_local(); + } + Err(error) => { + state.emit(ShadowServiceEventV1::Input { + kind: "recovery", + outcome: "rejected", + }); + if is_fatal_core_error(&error) { + state.mark_fatal(error); + } else { + state.reject(Some(peer), error); + } + } + } + } ShadowServiceMessageV1::CarrierSyncRequest { peer, request } => { if let Err(error) = state.validate_peer(peer) { state.reject(Some(peer), error); @@ -5894,6 +6035,12 @@ mod tests { response, } } + NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( + response, + ) => ShadowServiceMessageV1::CarrierEnvelopeResponse { + peer: sender as AuthorityIndex, + response, + }, NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { *sync_requests = sync_requests.saturating_add(1); ShadowServiceMessageV1::CarrierSyncRequest { @@ -6052,6 +6199,14 @@ mod tests { [recipient] .carrier_response(sender as AuthorityIndex, response) .unwrap(), + NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( + response, + ) => handles[recipient] + .carrier_envelope_response( + sender as AuthorityIndex, + response, + ) + .unwrap(), NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { handles[recipient] .carrier_sync_request( @@ -8176,6 +8331,43 @@ mod tests { stop(receiver, receiver_events, receiver_task).await; } + #[tokio::test] + async fn relayed_authenticated_candidate_serves_its_exact_persisted_envelope() { + let harness = Harness::new(); + let target = round_one_candidate(0, &harness.committee, 0x22); + let envelope = harness.envelope(&target, 0); + let (holder, mut events, task) = harness.start(1, Vec::new()); + wait_ready(&mut events).await; + holder.carrier(0, envelope.clone()).unwrap(); + loop { + if let ShadowServiceEventV1::Input { + kind: "carrier", + outcome: "authenticated", + } = next_event(&mut events).await + { + break; + } + } + + holder.carrier_request(2, target.reference()).unwrap(); + loop { + if let ShadowServiceEventV1::Network { + recipient: 2, + message: NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(response), + } = next_event(&mut events).await + { + assert_eq!(response.reference, target.reference()); + assert_eq!(response.canonical_carrier, envelope.canonical_carrier); + assert_eq!( + response.authentication_sidecar, + envelope.authentication_sidecar + ); + break; + } + } + stop(holder, events, task).await; + } + #[tokio::test] async fn poisoned_application_payload_waits_for_exact_delivery() { let harness = Harness::new(); @@ -8552,6 +8744,83 @@ mod tests { stop(handle, events, task).await; } + #[tokio::test] + async fn vector_bearing_recovery_authenticates_relay_or_falls_back_without_blame() { + for poisoned_receiver_entry in [false, true] { + let harness = Harness::new(); + let (handle, mut events, task) = harness.start(3, Vec::new()); + wait_ready(&mut events).await; + handle.peer_connected(0).unwrap(); + handle.peer_connected(1).unwrap(); + let target = round_one_candidate(2, &harness.committee, 0x63); + for sender in [0, 1] { + let outer = phase_carrier( + sender, + RbcPhaseStatementV1::Ready { + target: target.reference(), + }, + &harness.committee, + ); + handle + .carrier(sender, harness.envelope(&outer, sender)) + .unwrap(); + } + let holder = loop { + if let ShadowServiceEventV1::Network { + recipient, + message: NetworkMessage::RbcDagShadowCarrierRequest(reference), + } = next_event(&mut events).await + { + assert_eq!(reference, target.reference()); + break recipient; + } + }; + assert!(holder == 0 || holder == 1); + + let envelope = harness.envelope(&target, 2); + let mut authentication_sidecar = envelope.authentication_sidecar; + if poisoned_receiver_entry { + authentication_sidecar[3 + 3 * MAC_TAG_SIZE] ^= 1; + } + handle + .carrier_envelope_response( + holder, + RbcDagShadowCarrierEnvelopeResponse { + reference: target.reference(), + canonical_carrier: envelope.canonical_carrier, + authentication_sidecar, + }, + ) + .unwrap(); + + let expected_outcome = if poisoned_receiver_entry { + "accepted_content_only" + } else { + "accepted_authenticated" + }; + let mut accepted = false; + let mut delivered = false; + while !accepted || !delivered { + match next_event(&mut events).await { + ShadowServiceEventV1::Input { + kind: "recovery", + outcome, + } if outcome == expected_outcome => accepted = true, + ShadowServiceEventV1::Delivered(identity) => { + assert_eq!(identity.author, 2); + assert_eq!(identity.round, 1); + delivered = true; + } + ShadowServiceEventV1::Rejected { peer, error } => { + panic!("vector-bearing recovery assigned blame to {peer:?}: {error}") + } + _ => {} + } + } + stop(handle, events, task).await; + } + } + #[test] fn verified_payload_callback_coalesces_when_notification_queue_is_full() { let harness = Harness::new(); diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 98cccb09..1e1ea3ad 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -358,12 +358,16 @@ may attach different vectors to the same content reference, including a vector w one recipient and garbage for another. Correctness therefore depends only on the local entry and on the embedded four-phase protocol, never on agreement about the vector bytes. -Each node persists one exact vector variant with its carrier for restart and relay. The preference -order is locally generated, directly author-received, then first relayed variant with a valid local -entry. The implementation does not merge unverified entries from different vectors. Exact carrier -recovery after phase evidence may return canonical content without a vector; that recovery can -unblock phase progress, authoritative delivery, and READY certification, but it does not create -authenticated carrier admission or fast-clock stake. +Each node persists one exact vector variant with an authenticated carrier for restart and relay: +the locally generated sidecar for its own carrier, otherwise the first inbound variant whose local +entry verifies. The implementation never replaces that variant by arrival provenance and never +merges entries from different vectors. Exact carrier recovery after phase evidence prefers a new, +appended envelope-response message carrying the holder's persisted variant. If the requester's +local entry verifies, the response follows the ordinary relayed-ingress predicate and may create +authenticated admission, ECHO, and fast-clock stake. If the entry is absent, malformed, or invalid, +the same canonical bytes retain the legacy content-only authority and can still unblock phase +progress, authoritative delivery, and READY certification. The frozen content-only response remains +accepted for compatibility. A failed MAC check assigns no blame to either carrier author or holder. Public-signature modes use the same context-bound carrier statement without a recipient field and the same embedded RBC/consensus logic. They exist for controlled performance comparison, not as @@ -550,12 +554,14 @@ union of phase senders as candidate holders, and request the target from those h content is accepted only when canonical decoding recomputes the requested `BlockReference` and the context/committee checks succeed. Retention precedes any new local phase lock. -Recovery request/response is out-of-band byte transfer, not quorum testimony, admission, or a new -phase. A valid response can satisfy an already allocated evidence obligation but cannot create one. -The prototype retries recorded holders after GST. Its separate carrier catch-up mechanism requests -one exact `(author, round)` at a time and serves only retained locally authored outbound bytes; it -does not transfer ranges, certificates, checkpoints, committed observer history, or arbitrary late -state. +Recovery request/response is out-of-band byte transfer, not quorum testimony or a new phase. A +response can satisfy only an already allocated evidence obligation and cannot create one. A +vector-bearing response additionally grants ordinary carrier admission only when the exact +receiver-specific authenticator verifies under the same committee/context predicate as proactive +relayed ingress; otherwise it is content-only. The prototype retries recorded holders after GST. +Its separate carrier catch-up mechanism requests one exact `(author, round)` at a time and serves +only retained locally authored outbound bytes; it does not transfer ranges, certificates, +checkpoints, committed observer history, or arbitrary late state. ### 8.3 Batching and fairness @@ -955,6 +961,9 @@ The executable model and composed runtime tests should cover at minimum: - `M` ECHO to VOTE, `C` ECHO-or-VOTE to ACK, `C` ACK to READY, `O` authoritative delivery, and independent `Q`-READY certification; - evidence-before-content recovery from phase holders, including VOTE/ACK without local admission; +- vector-bearing phase-holder recovery that grants normal relayed admission only for a valid local + authenticator entry, falls back without blame for poisoned variants, preserves the frozen + content-only response, and replays the exact relayed provenance and sidecar after restart; - zero application load with heartbeat-only RBC completion; - independent logical-C2 timeout scheduling without changing the physical heartbeat, plus coalesced producer notification ordering in which an already-published application wins a newly @@ -1003,8 +1012,8 @@ The first fair benchmark matrix includes: - Sailfish++ as a certified signature-free comparison. Hold committee, load, transaction size, topology, latency injection, dissemination fanout, duration, -timeouts, and build constant. Report carrier, vector, ECHO, VOTE, ACK, READY, recovery, payload, -and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery +timeouts, and build constant. Report carrier, vector, ECHO, VOTE, ACK, READY, content-only recovery, +vector-bearing recovery, payload, and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained state. From 4459052263c591d851f328d85e1f556a5be17005 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:44:13 +0200 Subject: [PATCH 40/62] Measure RBC-DAG vertex carrier placement --- crates/starfish-core/src/metrics.rs | 18 ++++++++++ .../src/starfish_rbc_dag_shadow_service.rs | 36 +++++++++++++------ docs/starfish-rbc-dag-protocol.md | 5 +++ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 916c6fc7..564fdfd9 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -2442,6 +2442,24 @@ impl Metrics { shadow_input_count("consensus_vertex", "omitted"), ) ]); + let consensus_carrier_breakdown = |kind: &str| { + format!( + "bootstrap/C1/C2/C3/omitted={}/{}/{}/{}/{}", + shadow_input_count(kind, "bootstrap"), + shadow_input_count(kind, "c1"), + shadow_input_count(kind, "c2"), + shadow_input_count(kind, "c3"), + shadow_input_count(kind, "omitted"), + ) + }; + table.add_row(row![ + b->"Logical vertices by carrier kind:", + format!( + "application [{}], control/phase [{}]", + consensus_carrier_breakdown("application_consensus_vertex"), + consensus_carrier_breakdown("control_consensus_vertex"), + ) + ]); let stage_latency = RBC_DAG_PIPELINE_LATENCY_STAGES .iter() .map(|stage| { diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 0b9842c9..389ee9a4 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -3387,21 +3387,35 @@ impl ShadowServiceStateV1 { }, outcome: "accepted", }); + let consensus_vertex_outcome = if !fixed_consensus_vertex { + "omitted" + } else if consensus_slot == 1 { + "bootstrap" + } else if c1_ready { + "c1" + } else if c3_ready { + "c3" + } else if allow_no_vote { + "c2" + } else { + "unexpected" + }; self.emit(ShadowServiceEventV1::Input { kind: "consensus_vertex", - outcome: if !fixed_consensus_vertex { - "omitted" - } else if consensus_slot == 1 { - "bootstrap" - } else if c1_ready { - "c1" - } else if c3_ready { - "c3" - } else if allow_no_vote { - "c2" + outcome: consensus_vertex_outcome, + }); + // Keep the existing aggregate stable while exposing whether + // an available logical vertex was fixed on an application + // carrier or spent on a control/phase carrier. This is an + // event-local benchmark diagnostic and does not affect the + // carrier, journal, or consensus bytes. + self.emit(ShadowServiceEventV1::Input { + kind: if application_round.is_some() { + "application_consensus_vertex" } else { - "unexpected" + "control_consensus_vertex" }, + outcome: consensus_vertex_outcome, }); self.report_wal_delta(before); if let Some(reference) = assigned_application { diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 1e1ea3ad..57175444 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1017,6 +1017,11 @@ vector-bearing recovery, payload, and synchronization bytes separately. Also rep latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained state. +The benchmark also separates logical-vertex outcomes by enclosing carrier kind. Application and +control/phase carriers each report bootstrap, C1, C2, C3, or omitted without adding per-slot metric +labels. This event-local diagnostic distinguishes pre-inclusion scheduling delay from later +projection/decision latency without changing carrier, journal, or wire bytes. + Batching can reduce the number of separately scheduled RBC control messages, but it does not remove their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout From f0ec1457f99a511fc0f41b9ddfde6f4486d2d675 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:46:41 +0200 Subject: [PATCH 41/62] Add flagged RBC-DAG vote-QC fast path --- README.md | 9 ++ crates/starfish-core/src/config.rs | 11 +++ crates/starfish-core/src/net_sync.rs | 1 + .../src/starfish_rbc_dag/projection.rs | 16 +++- .../src/starfish_rbc_dag_shadow.rs | 84 ++++++++++++++++++- .../src/starfish_rbc_dag_shadow_service.rs | 11 +++ crates/starfish-core/src/validator.rs | 77 +++++++++++++++++ crates/starfish/src/main.rs | 18 ++++ docs/starfish-rbc-dag-protocol.md | 7 ++ 9 files changed, 230 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b39e91ba..d1aa5652 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,12 @@ single-slot exact synchronization rather than checkpoint or proof-safe late-node Its current authoritative journal uses the V4 autonomous WAL namespace with `SRD5` raw records so older traces cannot be reinterpreted under the optimistic-delivery rules. +Strict two-level Starfish finality remains the default. The local benchmark additionally exposes +`--starfish-rbc-dag-vote-qc-fast-path`, a deliberately flagged testbed experiment that commits an +exact projected leader as soon as its projected vote quorum is present instead of waiting for the +second certifier wave. It sends no additional protocol messages, but changes the proof shape and +must not be reported as the strict or production Starfish result. + The default WAL syncs every transition. `--starfish-rbc-dag-shadow-buffered-wal` preserves ordered frames but syncs only on clean shutdown and is not crash-safe. Actor replay covers the state explicitly documented in the protocol design; full validator crash recovery, bounded checkpoint @@ -354,6 +360,9 @@ cargo run --release --bin starfish -- local-benchmark \ The buffered WAL is benchmark-only and is not crash-safe. +To measure the separately labelled testbed fast path, append +`--starfish-rbc-dag-vote-qc-fast-path`. Omitting it always measures the strict two-level rule. + ### Local dryrun with monitoring and dashboard The dryrun script launches a Docker-based local testbed with diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 6c90f2c4..3b23e637 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -88,6 +88,12 @@ pub struct NodeParameters { /// transport but its phase messages cannot mark a block clean. #[serde(default)] pub starfish_rbc_dag_embedded_rbc_authority: bool, + /// Testbed-only optimistic commit path: finalize an exact projected + /// leader after its projected vote quorum, without waiting for the second + /// certifier quorum. Disabled by default because this changes the strict + /// Starfish finality proof shape. + #[serde(default)] + pub starfish_rbc_dag_vote_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but /// calls `sync_all` only during clean shutdown. This removes persistence /// pressure from latency experiments and deliberately forfeits the @@ -173,6 +179,7 @@ impl Default for NodeParameters { starfish_rbc_dag_autonomous_clock: false, starfish_rbc_dag_consensus_timeout: None, starfish_rbc_dag_embedded_rbc_authority: false, + starfish_rbc_dag_vote_qc_fast_path: false, starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: @@ -447,6 +454,7 @@ mod tests { assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert_eq!(parameters.starfish_rbc_dag_consensus_timeout, None); + assert!(!parameters.starfish_rbc_dag_vote_qc_fast_path); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); @@ -461,6 +469,7 @@ mod tests { assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert_eq!(decoded.starfish_rbc_dag_consensus_timeout, None); + assert!(!decoded.starfish_rbc_dag_vote_qc_fast_path); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); } @@ -471,6 +480,7 @@ mod tests { starfish_rbc_dag_autonomous_clock: true, leader_timeout: Duration::from_millis(125), starfish_rbc_dag_consensus_timeout: Some(Duration::from_millis(75)), + starfish_rbc_dag_vote_qc_fast_path: true, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -479,6 +489,7 @@ mod tests { let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); assert!(decoded.starfish_rbc_dag_shadow); assert!(decoded.starfish_rbc_dag_autonomous_clock); + assert!(decoded.starfish_rbc_dag_vote_qc_fast_path); assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(decoded.leader_timeout, Duration::from_millis(125)); assert_eq!( diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index f78cb841..7d2a72b2 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -3253,6 +3253,7 @@ impl NetworkSyncer Arc::clone(&metrics), rbc_dag_frontier_recovery_cursor, !rbc_dag_clock_start_paused, + node_parameters.starfish_rbc_dag_vote_qc_fast_path, ) } else { let start = if rbc_dag_clock_start_paused { diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 71950faa..24c80588 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -1156,9 +1156,19 @@ impl CertifiedProjectionModel { let Some(projected) = self.vertices.get(&certifier) else { return false; }; - let voter_authors: BTreeSet<_> = projected - .vertex - .strong_parents() + self.strong_parents_certify(projected.vertex.strong_parents(), leader) + } + + pub(crate) fn leader_values(&self, slot: LeaderSlotV1) -> Vec { + self.slot_values(slot.author, slot.round) + } + + fn strong_parents_certify( + &self, + strong_parents: &[ConsensusVertexReference], + leader: ConsensusVertexReference, + ) -> bool { + let voter_authors: BTreeSet<_> = strong_parents .iter() .filter_map(|parent| { self.vertices.get(parent).and_then(|voter| { diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 0e7e5737..6b9b29d4 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -743,6 +743,7 @@ pub(crate) struct StarfishRbcDagShadowV1 { pending_projected_vertices: Vec, pending_projection_decisions: Vec, pending_committed_frontiers: Vec, + vote_qc_fast_path: bool, poisoned: bool, } @@ -790,6 +791,7 @@ impl StarfishRbcDagShadowV1 { authorizer, wal_sync_policy, ShadowFrontierRecoveryPolicyV1::Observational, + false, ) } @@ -805,6 +807,7 @@ impl StarfishRbcDagShadowV1 { authorizer: ShadowAuthorizerV1, wal_sync_policy: ShadowWalSyncPolicyV1, recovery_cursor: Option, + vote_qc_fast_path: bool, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { Self::open_with_frontier_recovery_policy( path, @@ -814,6 +817,7 @@ impl StarfishRbcDagShadowV1 { authorizer, wal_sync_policy, ShadowFrontierRecoveryPolicyV1::Authoritative(recovery_cursor), + vote_qc_fast_path, ) } @@ -826,6 +830,7 @@ impl StarfishRbcDagShadowV1 { authorizer: ShadowAuthorizerV1, wal_sync_policy: ShadowWalSyncPolicyV1, frontier_recovery_policy: ShadowFrontierRecoveryPolicyV1, + vote_qc_fast_path: bool, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { validate_configuration(&committee, own_authority, context, &authorizer)?; let committee_size = committee.committee().len(); @@ -874,6 +879,7 @@ impl StarfishRbcDagShadowV1 { pending_projected_vertices: Vec::new(), pending_projection_decisions: Vec::new(), pending_committed_frontiers: Vec::new(), + vote_qc_fast_path, poisoned: false, }; @@ -2029,6 +2035,39 @@ impl StarfishRbcDagShadowV1 { Ok(()) } + /// Commit the longest consecutive prefix backed by an exact projected + /// vote quorum. Two conflicting vote quorums, or a vote quorum and a + /// negative-choice skip quorum, intersect in honest stake; an honest + /// logical author fixes exactly one choice. Reliable projected delivery + /// then makes the certificate eventually visible to every honest node. + /// This removes the redundant second certificate wave from the optimistic + /// direct-commit path while leaving skip and indirect recovery unchanged. + fn drive_vote_quorum_committer(&mut self) -> Result<(), ShadowErrorV1> { + loop { + let slot = self + .projection + .leader_slot(self.next_undecided_consensus_round); + let mut certified = Vec::new(); + for leader in self.projection.leader_values(slot) { + if self.projection.vote_stake(leader)? + >= self.committee.committee().quorum_threshold() + { + certified.push(leader); + } + } + if certified.len() > 1 { + return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues(slot).into()); + } + let Some(leader) = certified.pop() else { + return Ok(()); + }; + let decision = ProjectionDecisionV1::DirectCommit { leader }; + self.commit_projected_anchor(leader)?; + self.record_projection_decision(decision); + self.next_undecided_consensus_round = slot.round.saturating_add(1); + } + } + fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { if self.projected_decisions.insert(decision) { let slot = projection_decision_slot(decision); @@ -2360,7 +2399,11 @@ impl StarfishRbcDagShadowV1 { } self.activate_promised_references(); self.drive_promised_projection(); - self.drive_certified_projection() + self.drive_certified_projection()?; + if self.vote_qc_fast_path { + self.drive_vote_quorum_committer()?; + } + Ok(()) } fn decode_batch(&self, records: &[Vec]) -> Result { @@ -3797,6 +3840,45 @@ mod tests { [older, first_anchor, later_anchor] } + #[test] + fn vote_quorum_commits_before_the_certifier_round_projects() { + let mut network = TestNetwork::new(); + let node = &mut network.nodes[0]; + assert!(!node.vote_qc_fast_path, "strict finality is the default"); + node.vote_qc_fast_path = true; + let leader_author = node.committee.committee().elect_leader(1); + let leader = ordered_committer_vertex(leader_author, 1, 0); + node.projection.inject_projected_for_test( + leader, + Vec::new(), + LeaderChoiceV1::NoVote { + leader_author, + leader_round: 0, + }, + ); + for author in 0..3 as AuthorityIndex { + node.projection.inject_projected_for_test( + ordered_committer_vertex(author, 2, 0), + vec![leader], + LeaderChoiceV1::Vote { leader }, + ); + } + + let slot = node.projection.leader_slot(1); + assert_eq!( + node.projection.direct_decision(slot).unwrap(), + ProjectionDecisionV1::Undecided { slot }, + "the legacy two-level rule still waits for round-three certifiers" + ); + node.drive_vote_quorum_committer().unwrap(); + assert_eq!( + node.drain_projection_decisions(), + vec![ProjectionDecisionV1::DirectCommit { leader }] + ); + assert_eq!(node.drain_committed_frontiers().len(), 1); + assert_eq!(node.next_undecided_consensus_round, 2); + } + #[test] fn ordered_committer_is_deterministic_across_direct_anchor_arrival_orders() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 389ee9a4..34ea11a8 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -1227,6 +1227,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( None, None, true, + false, ) } @@ -1261,6 +1262,7 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( Some(metrics), None, true, + false, ) } @@ -1298,6 +1300,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( None, None, true, + false, ) } @@ -1336,6 +1339,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( Some(metrics), None, true, + false, ) } @@ -1378,6 +1382,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_paused_with_metric Some(metrics), None, false, + false, ) } @@ -1399,6 +1404,7 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 metrics: Arc, recovery_cursor: Option, clock_starts_active: bool, + vote_qc_fast_path: bool, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -1426,6 +1432,7 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 Some(metrics), recovery_cursor, clock_starts_active, + vote_qc_fast_path, ) } @@ -1495,6 +1502,7 @@ fn spawn_consensus_timeout_deadline_task( }); } +#[allow(clippy::too_many_arguments)] fn start_starfish_rbc_dag_shadow_service_with_mode_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, @@ -1508,6 +1516,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( metrics: Option>, recovery_cursor: Option, clock_starts_active: bool, + vote_qc_fast_path: bool, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -1689,6 +1698,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( authorizer, wal_sync_policy, recovery_cursor, + vote_qc_fast_path, ) } else { StarfishRbcDagShadowV1::open_with_wal_sync_policy( @@ -5764,6 +5774,7 @@ mod tests { None, None, false, + false, ) .unwrap() } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 77518c4e..d8c014ab 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -134,6 +134,22 @@ impl Validator { "Starfish-RBC-DAG embedded RBC authority requires the autonomous RBC-DAG shadow" )); } + if public_config.parameters.starfish_rbc_dag_vote_qc_fast_path + && !public_config + .parameters + .starfish_rbc_dag_embedded_rbc_authority + { + return Err(eyre!( + "Starfish-RBC-DAG vote-QC fast path requires embedded RBC-DAG authority" + )); + } + if public_config.parameters.starfish_rbc_dag_vote_qc_fast_path + && parameters.benchmark_duration.is_none() + { + return Err(eyre!( + "Starfish-RBC-DAG vote-QC fast path is restricted to finite testbed benchmarks" + )); + } if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { return Err(eyre!( "Starfish-RBC-DAG autonomous clock requires consensus 'starfish-rbc'" @@ -555,6 +571,67 @@ mod smoke_tests { })); } + #[tokio::test] + async fn vote_qc_fast_path_requires_embedded_authority() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_vote_qc_fast_path = true; + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("vote-QC fast path requires embedded RBC-DAG authority") + })); + } + + #[tokio::test] + async fn vote_qc_fast_path_requires_a_finite_benchmark() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_shadow = true; + public_config.parameters.starfish_rbc_dag_autonomous_clock = true; + public_config + .parameters + .starfish_rbc_dag_embedded_rbc_authority = true; + public_config.parameters.starfish_rbc_dag_vote_qc_fast_path = true; + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("vote-QC fast path is restricted to finite testbed benchmarks") + })); + } + #[tokio::test] async fn buffered_shadow_wal_requires_shadow_mode() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index d1f9dc88..bbd1091e 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -205,6 +205,11 @@ enum Operation { /// headers. Requires the autonomous RBC-DAG mode. #[clap(long, default_value_t = false)] starfish_rbc_dag_embedded_rbc_authority: bool, + /// Testbed-only: directly commit a projected leader after its exact + /// projected vote quorum, skipping the strict second certifier wave. + /// Requires embedded RBC-DAG authority and changes the finality proof. + #[clap(long, default_value_t = false)] + starfish_rbc_dag_vote_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. /// The physical carrier heartbeat remains on `leader_timeout`. #[clap(long, value_name = "INT")] @@ -331,6 +336,7 @@ async fn main() -> Result<()> { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_vote_qc_fast_path, starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, duration_secs, @@ -348,6 +354,7 @@ async fn main() -> Result<()> { node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; node_parameters.starfish_rbc_dag_embedded_rbc_authority = starfish_rbc_dag_embedded_rbc_authority; + node_parameters.starfish_rbc_dag_vote_qc_fast_path = starfish_rbc_dag_vote_qc_fast_path; node_parameters.starfish_rbc_dag_consensus_timeout = starfish_rbc_dag_consensus_timeout_ms.map(Duration::from_millis); node_parameters.starfish_rbc_dag_shadow_buffered_wal = @@ -590,6 +597,14 @@ async fn local_benchmark( .unwrap_or(node_parameters.leader_timeout) .as_millis() ); + println!( + "Vote-QC direct commit: {}", + if node_parameters.starfish_rbc_dag_vote_qc_fast_path { + "ENABLED (testbed-only; strict certifier wave skipped)" + } else { + "disabled (strict two-level Starfish finality)" + } + ); } if let Some(latency) = node_parameters.uniform_latency_ms { println!("Network Latency: {latency} ms (uniform)"); @@ -1522,6 +1537,7 @@ mod tests { "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-embedded-rbc-authority", + "--starfish-rbc-dag-vote-qc-fast-path", "--starfish-rbc-dag-consensus-timeout-ms", "250", "--starfish-rbc-dag-shadow-buffered-wal", @@ -1536,6 +1552,7 @@ mod tests { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_vote_qc_fast_path, starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, port_offset, @@ -1549,6 +1566,7 @@ mod tests { assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_dag_embedded_rbc_authority); + assert!(starfish_rbc_dag_vote_qc_fast_path); assert_eq!(starfish_rbc_dag_consensus_timeout_ms, Some(250)); assert!(starfish_rbc_dag_shadow_buffered_wal); assert_eq!(port_offset, 2500); diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 57175444..141b4cdd 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -714,6 +714,13 @@ The existing Starfish patterns are then evaluated from these explicit choices: - `Q` distinct certifier authors provide the direct-commit condition; and - a per-candidate quorum of explicit negative choices provides the direct-skip pattern. +This two-level rule is the strict default. The local testbed can explicitly enable +`--starfish-rbc-dag-vote-qc-fast-path`, which directly commits an exact projected leader after the +preceding projected vote quorum and therefore skips the second certifier wave. The experiment adds +no wire messages, retains the ordinary skip and indirect-recovery paths, and is intentionally +labelled separately because it changes the proof shape described by this section; it is not a +production-finality claim. + If the leader produces no value, `Q` immutable `NoVote(slot)` choices are a self-contained direct skip witness. If a Byzantine leader equivocates, `Vote(L)` is negative evidence for every other candidate, and the current Starfish per-candidate evaluator decides whether the collected explicit From e64b3123f0ec734537c8d2399dfd0a758979d75c Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:02:57 +0200 Subject: [PATCH 42/62] Raise RBC-DAG testbed committee bound --- .../src/starfish_rbc_dag_shadow_service.rs | 33 +++++++++++-------- docs/starfish-rbc-dag-protocol.md | 8 ++--- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 34ea11a8..99c23b85 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -64,17 +64,18 @@ use crate::{ // A mirror run must absorb one complete committee fan-in plus a small reserve; // autonomous repair additionally budgets a simultaneous request and response -// per peer. At the four-MiB carrier cap, allowing at most 64 queued inputs also -// caps carrier payload retention at 256 MiB (plus bounded sidecars and -// allocator overhead). This permits 60 mirror validators or 20 autonomous -// validators. Larger committees are rejected for this benchmark prototype +// per peer. At the four-MiB carrier cap, allowing at most 128 queued inputs +// caps carrier payload retention at 512 MiB (plus bounded sidecars and +// allocator overhead). This permits 124 mirror validators or 42 autonomous +// validators, including the 40-validator comparison profile. Larger committees +// are rejected for this benchmark prototype // instead of silently under-sizing the queue and reporting incomparable // results. // Use the full bounded allowance even for a small committee. A single fan-in // reserve is insufficient when several round bursts arrive while the actor is // synchronously making the previous transition durable. const SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1: usize = 64; -const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 64; +const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 128; const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); @@ -96,7 +97,10 @@ const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_milli /// frame ceiling is larger; a dedicated configurable payload limit remains a /// deployment-hardening boundary. Keeping only a bounded recent window stops /// unsolicited sidecars from turning the actor into an unbounded cache. -const SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1: usize = SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1; +// Keep the materialized-payload/quarantine cache at its original independent +// bound. Raising the actor's message burst allowance for n=40 must not double +// retained application state as a side effect. +const SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1: usize = 64; const SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1: usize = MAX_CARRIER_CONTENT_SIZE_V1; const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); /// Bound healthy physical-carrier production independently of actor/network @@ -8999,8 +9003,8 @@ mod tests { } #[tokio::test] - async fn sixty_validator_burst_fits_before_the_actor_drains() { - const LARGE_N: usize = 60; + async fn maximum_mirror_burst_fits_before_the_actor_drains() { + const LARGE_N: usize = 124; let input_capacity = shadow_input_capacity(LARGE_N, ShadowServiceModeV1::DirectMirror).unwrap(); assert_eq!(input_capacity, SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1); @@ -9044,17 +9048,18 @@ mod tests { } #[test] - fn autonomous_burst_budget_accepts_twenty_and_rejects_twenty_one() { + fn autonomous_burst_budget_accepts_forty_two_and_rejects_forty_three() { let mode = ShadowServiceModeV1::AutonomousClock { heartbeat_interval: Duration::from_millis(250), }; - assert_eq!(shadow_input_capacity(20, mode).unwrap(), 64); + assert_eq!(shadow_input_capacity(40, mode).unwrap(), 122); + assert_eq!(shadow_input_capacity(42, mode).unwrap(), 128); assert!(matches!( - shadow_input_capacity(21, mode), + shadow_input_capacity(43, mode), Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { - committee_size: 21, - required_capacity: 65, - maximum_capacity: 64, + committee_size: 43, + required_capacity: 131, + maximum_capacity: 128, }) )); } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 141b4cdd..b5808f5a 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -878,12 +878,12 @@ newest current-process observation (`<= 4`). These are empirical benchmark cover asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as proof of a protocol failure. -The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a slow +The actor reserves a bounded queue of up to 128 entries so several fan-in bursts can wait behind a slow reference transition (including synchronous fsync in the crash-safe profile), capping queued -maximum-sized carrier bodies at 256 MiB (plus sidecars and allocator overhead). Mirror mode budgets +maximum-sized carrier bodies at 512 MiB (plus sidecars and allocator overhead). Mirror mode budgets one peer fan-in plus five local/control inputs and accepts -at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and -exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs +at most 124 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and +exact-slot response per peer plus five control inputs and accepts at most 42 validators. Larger runs are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, healthy proactive rounds receive a repair grace period, and exact synchronization is rate-limited per peer. Exact synchronization transfers only one requested `(author, round)` and only from the From 27bb05f35625e20bcd50ff458f32cb38e5a2be48 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:31:23 +0200 Subject: [PATCH 43/62] reduce RBC-DAG carrier-path overhead --- crates/starfish-core/src/net_sync.rs | 172 ++++++++++++++---- .../src/starfish_rbc_dag/model.rs | 81 +++++++-- .../src/starfish_rbc_dag_shadow.rs | 37 +++- .../src/starfish_rbc_dag_shadow_service.rs | 161 +++++++++++++--- 4 files changed, 360 insertions(+), 91 deletions(-) diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 7d2a72b2..9ec6ffe6 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -45,7 +45,7 @@ use crate::{ RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, UtilizationTimerVecExt, }, - network::{BlockBatch, Connection, Network, NetworkMessage, ShardPayload}, + network::{BlockBatch, Connection, Network, NetworkMessage, RbcDagShadowCarrier, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, sailfish_service::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, @@ -350,21 +350,32 @@ impl RbcDagOutboundMailboxV1 { } } + #[cfg(test)] fn enqueue( &self, message: NetworkMessage, - committee: &Committee, + committee: &RbcDagCommitteeContextV1, + ) -> Result { + self.enqueue_with_proactive_reference(message, committee, None) + } + + fn enqueue_with_proactive_reference( + &self, + message: NetworkMessage, + committee: &RbcDagCommitteeContextV1, + proactive_reference: Option, ) -> Result { if let Some(reason) = self.inner.state.lock().failure.clone() { return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); } - let (class, key) = match rbc_dag_outbound_classification(&message, committee) { - Ok(classification) => classification, - Err(error) => { - self.fail(&error); - return Err(error); - } - }; + let (class, key) = + match rbc_dag_outbound_classification(&message, committee, proactive_reference) { + Ok(classification) => classification, + Err(error) => { + self.fail(&error); + return Err(error); + } + }; let framed_bytes = match bincode::serialized_size(&message) .map_err(|error| RbcDagOutboundMailboxErrorV1::Serialization(error.to_string())) .and_then(|size| { @@ -452,27 +463,19 @@ impl RbcDagOutboundMailboxV1 { fn rbc_dag_outbound_classification( message: &NetworkMessage, - committee: &Committee, + committee: &RbcDagCommitteeContextV1, + proactive_reference: Option, ) -> Result<(RbcDagOutboundClassV1, RbcDagOutboundKeyV1), RbcDagOutboundMailboxErrorV1> { let priority = RbcDagOutboundClassV1::Priority; match message { NetworkMessage::RbcDagShadowCarrier(carrier) => { - let candidate = - CandidateCarrierV1::decode_wire(&carrier.canonical_carrier, committee, None) - .map_err(|error| { - RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) - })?; - let canonical = candidate.canonical_wire_bytes().map_err(|error| { - RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) - })?; - if canonical != carrier.canonical_carrier { - return Err(RbcDagOutboundMailboxErrorV1::InvalidProactive( - "non-canonical carrier wire".to_owned(), - )); - } + let reference = match proactive_reference { + Some(reference) => reference, + None => rbc_dag_proactive_reference(carrier, committee)?, + }; Ok(( RbcDagOutboundClassV1::Proactive, - RbcDagOutboundKeyV1::Proactive(candidate.reference()), + RbcDagOutboundKeyV1::Proactive(reference), )) } NetworkMessage::RbcDagShadowCarrierRequest(reference) => { @@ -506,6 +509,24 @@ fn rbc_dag_outbound_classification( } } +fn rbc_dag_proactive_reference( + carrier: &RbcDagShadowCarrier, + committee: &RbcDagCommitteeContextV1, +) -> Result { + let candidate = + CandidateCarrierV1::decode_wire_with_committee(&carrier.canonical_carrier, committee, None) + .map_err(|error| RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()))?; + let canonical = candidate + .canonical_wire_bytes() + .map_err(|error| RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()))?; + if canonical != carrier.canonical_carrier { + return Err(RbcDagOutboundMailboxErrorV1::InvalidProactive( + "non-canonical carrier wire".to_owned(), + )); + } + Ok(candidate.reference()) +} + fn rbc_dag_outbound_messages_equal(left: &NetworkMessage, right: &NetworkMessage) -> bool { match (left, right) { (NetworkMessage::RbcDagShadowCarrier(left), NetworkMessage::RbcDagShadowCarrier(right)) => { @@ -3188,6 +3209,10 @@ impl NetworkSyncer } else { (None, None, None) }; + let rbc_dag_committee_context = recovered_shadow_local_headers.as_ref().map(|_| { + RbcDagCommitteeContextV1::new(committee.clone()) + .expect("validated committee must initialize the RBC-DAG shadow") + }); let (starfish_rbc_dag_shadow_service, rbc_dag_shadow_event_rx, rbc_dag_shadow_service_task) = if let Some(recovered_local_headers) = recovered_shadow_local_headers { let protocol_instance_bytes = node_parameters @@ -3197,8 +3222,9 @@ impl NetworkSyncer protocol_instance_bytes, node_parameters.starfish_rbc_dag_autonomous_clock, ); - let committee_context = RbcDagCommitteeContextV1::new(committee.clone()) - .expect("validated committee must initialize the RBC-DAG shadow"); + let committee_context = rbc_dag_committee_context + .clone() + .expect("RBC-DAG runtime must retain its validated committee context"); let context = RbcDagContextV1::new_with_committee( protocol_instance, &committee_context, @@ -3617,12 +3643,20 @@ impl NetworkSyncer }; let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); + let rbc_dag_committee_context = rbc_dag_committee_context + .clone() + .expect("RBC-DAG event router must retain its validated committee context"); let shadow_metrics = metrics.clone(); let rbc_dag_clock_bridge_tx = rbc_dag_clock_bridge_tx.clone(); let rbc_dag_core_control_tx = rbc_dag_core_control_tx; let rbc_dag_assignment_tx = rbc_dag_assignment_tx; let rbc_dag_shutdown_started = rbc_dag_shutdown_started; handle.spawn(async move { + // A local broadcast emits the same canonical carrier once per + // recipient. Validate it on the first event and reuse only its + // exact reference while the bytes remain identical; receiver + // authentication and canonical decoding are unchanged. + let mut last_proactive_carrier: Option<(Vec, BlockReference)> = None; let mut router_guard = embedded_rbc_authority.then(|| { RbcDagEventRouterGuardV1::new( shadow_metrics.clone(), @@ -3651,7 +3685,49 @@ impl NetworkSyncer .get(&recipient) .cloned(); if let Some(mailbox) = mailbox { - match mailbox.enqueue(message, &event_inner.committee) { + let proactive_reference = match &message { + NetworkMessage::RbcDagShadowCarrier(carrier) => { + if let Some(reference) = last_proactive_carrier + .as_ref() + .filter(|(canonical, _)| { + canonical == &carrier.canonical_carrier + }) + .map(|(_, reference)| *reference) + { + Some(reference) + } else { + match rbc_dag_proactive_reference( + carrier, + &rbc_dag_committee_context, + ) { + Ok(reference) => { + last_proactive_carrier = Some(( + carrier.canonical_carrier.clone(), + reference, + )); + Some(reference) + } + Err(error) => { + mailbox.fail(&error); + fail_rbc_dag_outbound_transport( + &shadow_metrics, + rbc_dag_clock_bridge_tx.as_ref(), + embedded_rbc_authority, + recipient, + &error, + ); + continue; + } + } + } + } + _ => None, + }; + match mailbox.enqueue_with_proactive_reference( + message, + &rbc_dag_committee_context, + proactive_reference, + ) { Ok(RbcDagOutboundEnqueueV1::Added) => shadow_metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["network", "sent"]) @@ -5482,21 +5558,31 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_coalesces_exact_duplicates_and_rejects_conflicts() { let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(committee).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); assert_eq!( mailbox - .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) + .enqueue( + rbc_dag_outbound_test_sync_response(7, 0xA1), + &committee_context, + ) .unwrap(), RbcDagOutboundEnqueueV1::Added ); assert_eq!( mailbox - .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) + .enqueue( + rbc_dag_outbound_test_sync_response(7, 0xA1), + &committee_context, + ) .unwrap(), RbcDagOutboundEnqueueV1::Coalesced ); assert!(matches!( - mailbox.enqueue(rbc_dag_outbound_test_sync_response(7, 0xB1), &committee), + mailbox.enqueue( + rbc_dag_outbound_test_sync_response(7, 0xB1), + &committee_context, + ), Err(RbcDagOutboundMailboxErrorV1::ConflictingDuplicate { class: RbcDagOutboundClassV1::Priority, key: RbcDagOutboundKeyV1::SyncResponse(1, 7), @@ -5510,11 +5596,12 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_drains_priority_before_proactive() { let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(committee.clone()).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); let (reference, proactive) = rbc_dag_outbound_test_carrier(&committee, 11); - mailbox.enqueue(proactive, &committee).unwrap(); + mailbox.enqueue(proactive, &committee_context).unwrap(); mailbox - .enqueue(rbc_dag_outbound_test_sync_request(9), &committee) + .enqueue(rbc_dag_outbound_test_sync_request(9), &committee_context) .unwrap(); let (class, first) = mailbox.try_pop().unwrap(); @@ -5529,7 +5616,9 @@ mod tests { let (class, second) = mailbox.try_pop().unwrap(); assert_eq!(class, RbcDagOutboundClassV1::Proactive); assert!(matches!( - rbc_dag_outbound_classification(&second, &committee).unwrap().1, + rbc_dag_outbound_classification(&second, &committee_context, None) + .unwrap() + .1, RbcDagOutboundKeyV1::Proactive(actual) if actual == reference )); } @@ -5537,6 +5626,7 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_never_evicts_a_unique_proactive_reference() { let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(committee.clone()).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); let mut first_reference = None; for marker in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY { @@ -5544,7 +5634,7 @@ mod tests { rbc_dag_outbound_test_carrier(&committee, marker as TimestampNs); first_reference.get_or_insert(reference); assert_eq!( - mailbox.enqueue(message, &committee).unwrap(), + mailbox.enqueue(message, &committee_context).unwrap(), RbcDagOutboundEnqueueV1::Added ); } @@ -5553,7 +5643,7 @@ mod tests { STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as TimestampNs + 1, ); assert!(matches!( - mailbox.enqueue(overflow, &committee), + mailbox.enqueue(overflow, &committee_context), Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { class: RbcDagOutboundClassV1::Proactive, capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, @@ -5575,10 +5665,14 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_bounds_distinct_priority_keys_and_bytes() { let committee = Committee::new_test(vec![1; 4]); + let committee_context = RbcDagCommitteeContextV1::new(committee).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); for round in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber { mailbox - .enqueue(rbc_dag_outbound_test_sync_request(round), &committee) + .enqueue( + rbc_dag_outbound_test_sync_request(round), + &committee_context, + ) .unwrap(); } assert!(matches!( @@ -5586,7 +5680,7 @@ mod tests { rbc_dag_outbound_test_sync_request( STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber + 1, ), - &committee, + &committee_context, ), Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { class: RbcDagOutboundClassV1::Priority, @@ -5616,9 +5710,9 @@ mod tests { ), failure: None, }); - byte_bounded.enqueue(first, &committee).unwrap(); + byte_bounded.enqueue(first, &committee_context).unwrap(); assert!(matches!( - byte_bounded.enqueue(rbc_dag_outbound_test_sync_request(2), &committee), + byte_bounded.enqueue(rbc_dag_outbound_test_sync_request(2), &committee_context,), Err(RbcDagOutboundMailboxErrorV1::ByteCapacity { class: RbcDagOutboundClassV1::Priority, .. diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index b89b0642..ea8d2685 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -344,10 +344,38 @@ struct RbcCandidateState { votes: BTreeSet, acks: BTreeSet, readies: BTreeSet, + echo_stake: Stake, + vote_stake: Stake, + ack_stake: Stake, + ready_stake: Stake, requested_holders: BTreeSet, } impl RbcCandidateState { + fn insert_echo(&mut self, sender: AuthorityIndex, stake: Stake) { + if self.echoes.insert(sender) { + self.echo_stake = self.echo_stake.saturating_add(stake); + } + } + + fn insert_vote(&mut self, sender: AuthorityIndex, stake: Stake) { + if self.votes.insert(sender) { + self.vote_stake = self.vote_stake.saturating_add(stake); + } + } + + fn insert_ack(&mut self, sender: AuthorityIndex, stake: Stake) { + if self.acks.insert(sender) { + self.ack_stake = self.ack_stake.saturating_add(stake); + } + } + + fn insert_ready(&mut self, sender: AuthorityIndex, stake: Stake) { + if self.readies.insert(sender) { + self.ready_stake = self.ready_stake.saturating_add(stake); + } + } + fn holders(&self) -> BTreeSet { self.echoes .iter() @@ -1252,6 +1280,7 @@ impl RbcDagModel { fn authorize_local_echo(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; + let own_stake = self.authority_stake(own); if own == reference.authority { // The target author is excluded from ECHO/VOTE/ACK. A locally // fixed high-stake author instead seeds READY: its stake is at @@ -1270,8 +1299,7 @@ impl RbcDagModel { slot.candidates .entry(reference) .or_default() - .echoes - .insert(own); + .insert_echo(own, own_stake); let statement = RbcPhaseStatementV1::Echo { target: reference }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); @@ -1280,6 +1308,7 @@ impl RbcDagModel { fn authorize_local_ready(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; + let own_stake = self.authority_stake(own); let slot = self.rbc_slot_mut(reference); if slot.readied.is_some() { return; @@ -1289,8 +1318,7 @@ impl RbcDagModel { slot.candidates .entry(reference) .or_default() - .readies - .insert(own); + .insert_ready(own, own_stake); let statement = RbcPhaseStatementV1::Ready { target: reference }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); @@ -1406,6 +1434,7 @@ impl RbcDagModel { return; } } + let sender_stake = self.authority_stake(sender); let slot = self.rbc_slot_mut(target); let senders = match statement { RbcPhaseStatementV1::Echo { .. } => &mut slot.echo_by_sender, @@ -1423,16 +1452,16 @@ impl RbcDagModel { let candidate = slot.candidates.entry(target).or_default(); match statement { RbcPhaseStatementV1::Echo { .. } => { - candidate.echoes.insert(sender); + candidate.insert_echo(sender, sender_stake); } RbcPhaseStatementV1::Vote { .. } => { - candidate.votes.insert(sender); + candidate.insert_vote(sender, sender_stake); } RbcPhaseStatementV1::Ack { .. } => { - candidate.acks.insert(sender); + candidate.insert_ack(sender, sender_stake); } RbcPhaseStatementV1::Ready { .. } => { - candidate.readies.insert(sender); + candidate.insert_ready(sender, sender_stake); } } self.drive_rbc(target, log); @@ -1508,11 +1537,24 @@ impl RbcDagModel { .get(&slot_key) .and_then(|slot| slot.candidates.get(&target)) .expect("the candidate remains allocated"); + debug_assert_eq!( + candidate.echo_stake, + self.voters_stake_excluding(&candidate.echoes, target.authority) + ); + debug_assert_eq!( + candidate.vote_stake, + self.voters_stake_excluding(&candidate.votes, target.authority) + ); + debug_assert_eq!( + candidate.ack_stake, + self.voters_stake_excluding(&candidate.acks, target.authority) + ); + debug_assert_eq!(candidate.ready_stake, self.voters_stake(&candidate.readies)); ( - self.voters_stake_excluding(&candidate.echoes, target.authority), - self.voters_stake_excluding(&candidate.votes, target.authority), - self.voters_stake_excluding(&candidate.acks, target.authority), - self.voters_stake(&candidate.readies), + candidate.echo_stake, + candidate.vote_stake, + candidate.ack_stake, + candidate.ready_stake, ) }; let (vote_trigger, ack_trigger, optimistic_ready_trigger, promise_trigger) = thresholds @@ -1590,20 +1632,28 @@ impl RbcDagModel { } RbcAction::SendVote => { let own = self.own_authority; + let own_stake = self.authority_stake(own); let slot = self.rbc_slot_mut(target); slot.voted = Some(target); slot.vote_by_sender.insert(own, target); - slot.candidates.entry(target).or_default().votes.insert(own); + slot.candidates + .entry(target) + .or_default() + .insert_vote(own, own_stake); let statement = RbcPhaseStatementV1::Vote { target }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); } RbcAction::SendAck => { let own = self.own_authority; + let own_stake = self.authority_stake(own); let slot = self.rbc_slot_mut(target); slot.acked = Some(target); slot.ack_by_sender.insert(own, target); - slot.candidates.entry(target).or_default().acks.insert(own); + slot.candidates + .entry(target) + .or_default() + .insert_ack(own, own_stake); let statement = RbcPhaseStatementV1::Ack { target }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); @@ -3456,7 +3506,8 @@ mod tests { .carriers .insert(reference, CarrierRecord::new(carrier, false)); let mut candidate_state = RbcCandidateState::default(); - candidate_state.readies.extend([1, 2]); + candidate_state.insert_ready(1, 1); + candidate_state.insert_ready(2, 1); let mut slot = RbcSlotState::default(); slot.ready_by_sender .extend([(1, reference), (2, reference)]); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 6b9b29d4..321585a4 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -26,8 +26,9 @@ use crate::{ RbcPhaseStatementV1, carrier_genesis_reference, journal::{IngressProvenanceV1, JournalErrorV1, JournalEventV1, WriteAheadJournalV1}, model::{ - DeliveryPromiseBasisV1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, ModelError, - ModelInputRecord, ModelTraceEvent, RbcDagModel, + DeliveryPromiseBasisV1, EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, + EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, ModelError, ModelInputRecord, + ModelTraceEvent, RbcDagModel, }, projection::{ C1StrongParentWitnessV1, CertifiedProjectionError, CertifiedProjectionModel, @@ -997,12 +998,34 @@ impl StarfishRbcDagShadowV1 { /// window. These are bounded by the reducer's future-carrier window and /// become admitted only through sequential clock advancement. pub(crate) fn buffered_authenticated_carrier_count(&self) -> usize { - self.authenticated_slots - .iter() - .filter(|((authority, round), reference)| { - self.model.admitted_reference(*authority, *round) != Some(**reference) + let Some(first_buffered_round) = self + .local_carrier_round() + .checked_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) + .and_then(|round| round.checked_add(1)) + else { + return 0; + }; + let buffered = self + .committee + .committee() + .authorities() + .map(|authority| { + self.authenticated_slots + .range((authority, first_buffered_round)..=(authority, RoundNumber::MAX)) + .count() }) - .count() + .sum(); + debug_assert_eq!( + buffered, + self.authenticated_slots + .iter() + .filter(|((authority, round), reference)| { + self.model.admitted_reference(*authority, *round) != Some(**reference) + }) + .count(), + "the reducer must admit every authenticated carrier inside its admission window" + ); + buffered } pub(crate) fn drain_projection_decisions(&mut self) -> Vec { diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 99c23b85..d9ad8613 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -79,6 +79,8 @@ const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 128; const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); +const SHADOW_APPLICATION_SUBMISSION_GRACE_V1: Duration = Duration::from_millis(100); +const SHADOW_APPLICATION_SUBMISSION_GRACE_COMMITTEE_STEP_V1: usize = 10; const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); const SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1: Duration = Duration::from_millis(100); // At most sixteen distinct exact slots may bypass the duplicate retry interval @@ -97,10 +99,12 @@ const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_milli /// frame ceiling is larger; a dedicated configurable payload limit remains a /// deployment-hardening boundary. Keeping only a bounded recent window stops /// unsolicited sidecars from turning the actor into an unbounded cache. -// Keep the materialized-payload/quarantine cache at its original independent -// bound. Raising the actor's message burst allowance for n=40 must not double -// retained application state as a side effect. -const SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1: usize = 64; +// Bound the materialized-payload/quarantine cache independently from history, +// but size it for the same three-message-per-peer fan-in that the autonomous +// actor accepts. A fixed 64-entry callback map is insufficient at n=40: two +// adjacent carrier waves can complete verification while the actor applies +// the previous wave. The global 128 ceiling keeps the testbed bound explicit. +const SHADOW_APPLICATION_PAYLOAD_MIN_CAPACITY_V1: usize = 64; const SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1: usize = MAX_CARRIER_CONTENT_SIZE_V1; const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); /// Bound healthy physical-carrier production independently of actor/network @@ -110,6 +114,22 @@ const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_mi const SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1: u32 = 20; const SHADOW_NORMAL_CARRIER_MIN_SPACING_V1: Duration = Duration::from_millis(1); +fn shadow_application_payload_capacity(committee_size: usize) -> usize { + committee_size.saturating_mul(3).clamp( + SHADOW_APPLICATION_PAYLOAD_MIN_CAPACITY_V1, + SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1, + ) +} + +fn shadow_application_submission_grace(committee_size: usize) -> Duration { + let fan_in_groups = committee_size + .max(1) + .div_ceil(SHADOW_APPLICATION_SUBMISSION_GRACE_COMMITTEE_STEP_V1); + SHADOW_APPLICATION_SUBMISSION_GRACE_V1 + .checked_div(u32::try_from(fan_in_groups).unwrap_or(u32::MAX)) + .unwrap_or_default() +} + type CarrierSyncSlotV1 = (RoundNumber, AuthorityIndex); type DesiredCarrierSyncResponseV1 = (AuthorityIndex, RbcDagShadowCarrierSyncResponse); @@ -442,10 +462,9 @@ impl StarfishRbcDagShadowServiceHandleV1 { existing.acknowledge_assignment |= local.acknowledge_assignment; return Ok(()); } - if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - return Err(ShadowServiceErrorV1::ApplicationStateCapacity { - capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, - }); + let capacity = shadow_application_payload_capacity(self.committee_size); + if desired.len() >= capacity { + return Err(ShadowServiceErrorV1::ApplicationStateCapacity { capacity }); } desired.insert(local.round, local); drop(desired); @@ -742,10 +761,9 @@ impl StarfishRbcDagShadowServiceHandleV1 { } return Ok(()); } - if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - return Err(ShadowServiceErrorV1::ApplicationStateCapacity { - capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, - }); + let capacity = shadow_application_payload_capacity(self.committee_size); + if desired.len() >= capacity { + return Err(ShadowServiceErrorV1::ApplicationStateCapacity { capacity }); } desired.insert(application, payload); drop(desired); @@ -1908,7 +1926,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( for (carrier, header) in recovered_application_headers .into_iter() .rev() - .take(SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1) + .take(shadow_application_payload_capacity(committee_size)) { authorized_applications.insert( header.reference(), @@ -1975,6 +1993,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( #[cfg(test)] sync_max_desired_responses: 0, awaiting_application_submission: false, + application_submission_deadline: None, consensus_pacemaker, consensus_timeout, normal_carrier_min_spacing: mode @@ -2245,10 +2264,13 @@ struct ShadowServiceStateV1 { #[cfg(test)] sync_max_desired_responses: usize, /// A successful application-carrier assignment just released the core's - /// one-outstanding producer gate. Give that exact producer one actor turn - /// to submit its successor before C3 spends the next physical slot on a - /// control heartbeat. Maintenance/heartbeat messages bound the wait. + /// one-outstanding producer gate. Give that exact producer a bounded + /// maintenance epoch to submit its successor before any normal phase or + /// fallback work spends the next physical slot on a control carrier. + /// Repair traffic remains independently bounded and may bypass this + /// scheduling preference. awaiting_application_submission: bool, + application_submission_deadline: Option, consensus_pacemaker: ConsensusPacemakerV1, /// Logical C2 fallback deadline. This is independent of the physical /// heartbeat so experiments can vary consensus permission without @@ -2293,6 +2315,7 @@ impl ShadowServiceStateV1 { self.consensus_pacemaker = ConsensusPacemakerV1::new(self.core.next_local_consensus_round()); self.awaiting_application_submission = false; + self.application_submission_deadline = None; self.heartbeat_notification_pending .store(false, Ordering::Release); @@ -2403,8 +2426,9 @@ impl ShadowServiceStateV1 { } fn make_application_state_room(&mut self, application: BlockReference) { + let capacity = shadow_application_payload_capacity(self.committee_size); if self.authorized_applications.contains_key(&application) - || self.authorized_applications.len() < SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + || self.authorized_applications.len() < capacity { return; } @@ -2496,7 +2520,8 @@ impl ShadowServiceStateV1 { payload: Option>, ) -> Result<(), ShadowServiceErrorV1> { if !self.quarantined_application_payloads.contains_key(&carrier) - && self.quarantined_application_payloads.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + && self.quarantined_application_payloads.len() + >= shadow_application_payload_capacity(self.committee_size) { self.quarantined_application_payloads.pop_first(); } @@ -2772,6 +2797,7 @@ impl ShadowServiceStateV1 { let desired = std::mem::take(&mut *self.desired_local_applications.lock()); if !desired.is_empty() { self.awaiting_application_submission = false; + self.application_submission_deadline = None; } for local in desired.into_values() { self.enqueue_local(local); @@ -3241,6 +3267,7 @@ impl ShadowServiceStateV1 { return; } self.awaiting_application_submission = false; + self.application_submission_deadline = None; } fn cancel_normal_carrier_deadline(&mut self) { @@ -3434,6 +3461,8 @@ impl ShadowServiceStateV1 { self.report_wal_delta(before); if let Some(reference) = assigned_application { self.awaiting_application_submission = true; + self.application_submission_deadline = Instant::now() + .checked_add(shadow_application_submission_grace(self.committee_size)); self.emit(ShadowServiceEventV1::ApplicationAssigned(reference)); } if let Some(application) = application { @@ -3642,6 +3671,25 @@ impl ShadowServiceStateV1 { /// harmless idempotent replays. fn retry_pending_local(&mut self) { if self.mode.is_autonomous() { + // The producer is released only after its previous application + // carrier is durably fixed. Phase effects from that transition + // are processed synchronously and would otherwise consume the + // newly opened physical slot before the producer's successor can + // cross the ordered Core bridge. Piggyback those phases on the + // successor when it arrives. A deadline anchored at assignment + // retries the phase carrier after 100 ms, so a stalled producer + // cannot stop physical RBC progress. + if self.awaiting_application_submission && self.pending_local.is_empty() { + let now = Instant::now(); + if let Some(deadline) = self.application_submission_deadline { + if now < deadline { + self.schedule_normal_carrier_deadline(deadline); + return; + } + } + self.awaiting_application_submission = false; + self.application_submission_deadline = None; + } while (!self.pending_local.is_empty() || self.core.has_pending_application_phase_work()) && self.core.can_create_carrier() && !self.fatal @@ -4590,8 +4638,8 @@ fn run_shadow_service( } ShadowServiceMessageV1::TopologyChanged => {} ShadowServiceMessageV1::RetryRecovery => { - state.awaiting_application_submission = false; state.reconcile_local_applications(); + state.retry_pending_local(); state.reconcile_pending_recovery(); state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); @@ -4599,6 +4647,7 @@ fn run_shadow_service( } ShadowServiceMessageV1::HeartbeatTick => { state.awaiting_application_submission = false; + state.application_submission_deadline = None; state.try_create_autonomous_carrier(); } ShadowServiceMessageV1::NormalCarrierDeadline { generation } => { @@ -5870,6 +5919,7 @@ mod tests { sync_max_outstanding: 0, sync_max_desired_responses: 0, awaiting_application_submission: false, + application_submission_deadline: None, consensus_pacemaker: ConsensusPacemakerV1::new(slot), consensus_timeout: leader_timeout, // Existing state-level tests invoke creation synchronously and do @@ -6796,7 +6846,8 @@ mod tests { ); tokio::task::spawn_blocking(move || { let mut state = state; - for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + let capacity = shadow_application_payload_capacity(N); + for offset in 0..=capacity { let round = offset as RoundNumber + 1; let header = direct_header(0, round, offset as u8); state @@ -6810,12 +6861,9 @@ mod tests { ) .unwrap(); } - assert_eq!( - state.authorized_applications.len(), - SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 - ); + assert_eq!(state.authorized_applications.len(), capacity); - for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + for offset in 0..=capacity { let round = offset as RoundNumber + 1; state .quarantine_application( @@ -6826,10 +6874,7 @@ mod tests { ) .unwrap(); } - assert_eq!( - state.quarantined_application_payloads.len(), - SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 - ); + assert_eq!(state.quarantined_application_payloads.len(), capacity); state.core.shutdown().unwrap(); }) .await @@ -6855,7 +6900,7 @@ mod tests { tokio::task::spawn_blocking(move || { let mut state = state; let mut applications = Vec::new(); - for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + for offset in 0..=shadow_application_payload_capacity(N) { let round = offset as RoundNumber + 1; let header = direct_header(0, round, offset as u8); applications.push(header.reference()); @@ -7213,6 +7258,47 @@ mod tests { state.core.shutdown().unwrap(); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn application_submission_grace_is_anchored_bounded_and_coalesced() { + let harness = Harness::new(); + let (core, _) = StarfishRbcDagShadowV1::open( + &harness.paths[0], + harness.committee.clone(), + 0, + harness.context, + ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), + ) + .unwrap(); + let (mut state, _events, _message_rx, _message_tx) = standalone_autonomous_state( + core, + harness.committee.clone(), + Duration::from_secs(60 * 60), + ); + + let round = state.core.local_carrier_round(); + let deadline = Instant::now() + Duration::from_secs(60); + state.awaiting_application_submission = true; + state.application_submission_deadline = Some(deadline); + state.retry_pending_local(); + assert_eq!(state.core.local_carrier_round(), round); + assert_eq!( + state + .normal_carrier_deadline + .expect("the grace must reuse the coalesced carrier wake") + .deadline, + deadline + ); + + let generation = state.normal_carrier_generation; + state.retry_pending_local(); + assert_eq!(state.normal_carrier_generation, generation); + state.application_submission_deadline = Some(Instant::now()); + state.retry_pending_local(); + assert!(!state.awaiting_application_submission); + assert_eq!(state.application_submission_deadline, None); + state.core.shutdown().unwrap(); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn paused_autonomous_clock_uses_one_idempotent_fresh_activation_epoch() { let harness = Harness::new(); @@ -9052,6 +9138,21 @@ mod tests { let mode = ShadowServiceModeV1::AutonomousClock { heartbeat_interval: Duration::from_millis(250), }; + assert_eq!(shadow_application_payload_capacity(10), 64); + assert_eq!(shadow_application_payload_capacity(40), 120); + assert_eq!(shadow_application_payload_capacity(42), 126); + assert_eq!( + shadow_application_submission_grace(10), + Duration::from_millis(100) + ); + assert_eq!( + shadow_application_submission_grace(40), + Duration::from_millis(25) + ); + assert_eq!( + shadow_application_submission_grace(42), + Duration::from_millis(20) + ); assert_eq!(shadow_input_capacity(40, mode).unwrap(), 122); assert_eq!(shadow_input_capacity(42, mode).unwrap(), 128); assert!(matches!( From 73f19a084fbd7a9e0b9f338c325bf0bbc8abbc03 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:40:51 +0200 Subject: [PATCH 44/62] add single-DAG Starfish RBC prototype --- crates/starfish-core/src/broadcaster.rs | 3 + crates/starfish-core/src/config.rs | 9 + .../starfish-core/src/consensus/linearizer.rs | 1 + .../src/consensus/universal_committer.rs | 1 + crates/starfish-core/src/core.rs | 63 +++- .../starfish-core/src/core_thread/spawned.rs | 21 ++ crates/starfish-core/src/crypto.rs | 51 ++- crates/starfish-core/src/dag_state.rs | 91 ++++- crates/starfish-core/src/net_sync.rs | 28 +- crates/starfish-core/src/starfish_rbc.rs | 328 +++++++++++++++++- .../starfish-core/src/starfish_rbc_service.rs | 145 +++++++- crates/starfish-core/src/syncer.rs | 46 ++- crates/starfish-core/src/threshold_clock.rs | 1 + crates/starfish-core/src/types.rs | 233 ++++++++++++- crates/starfish-core/src/validator.rs | 22 ++ crates/starfish/src/main.rs | 27 +- docs/starfish-rbc-dag-protocol.md | 6 + docs/starfish-rbc-single-dag-v3.md | 127 +++++++ 18 files changed, 1152 insertions(+), 51 deletions(-) create mode 100644 docs/starfish-rbc-single-dag-v3.md diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index 881331f4..2a65ce78 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -88,6 +88,7 @@ impl BroadcasterParameters { }, ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::CordialMiners @@ -381,6 +382,7 @@ where match self.inner.dag_state.consensus_protocol { ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::SparseStarfishSpeed => { @@ -939,6 +941,7 @@ fn push_transport_format(consensus_protocol: ConsensusProtocol) -> PushOtherBloc match consensus_protocol { ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::SparseStarfishSpeed => PushOtherBlocksFormat::HeadersAndShards, diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 3b23e637..fad2d43e 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -94,6 +94,12 @@ pub struct NodeParameters { /// Starfish finality proof shape. #[serde(default)] pub starfish_rbc_dag_vote_qc_fast_path: bool, + /// Testbed-only single-DAG RBC path: deliver an exact header after quorum + /// ECHO rather than quorum READY. Quorum intersection preserves a unique + /// value, but selective Byzantine ECHO withholding can violate totality; + /// this must remain an explicit benchmark flag. + #[serde(default)] + pub starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but /// calls `sync_all` only during clean shutdown. This removes persistence /// pressure from latency experiments and deliberately forfeits the @@ -180,6 +186,7 @@ impl Default for NodeParameters { starfish_rbc_dag_consensus_timeout: None, starfish_rbc_dag_embedded_rbc_authority: false, starfish_rbc_dag_vote_qc_fast_path: false, + starfish_rbc_single_dag_echo_qc_fast_path: false, starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: @@ -455,6 +462,7 @@ mod tests { assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert_eq!(parameters.starfish_rbc_dag_consensus_timeout, None); assert!(!parameters.starfish_rbc_dag_vote_qc_fast_path); + assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); @@ -470,6 +478,7 @@ mod tests { assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert_eq!(decoded.starfish_rbc_dag_consensus_timeout, None); assert!(!decoded.starfish_rbc_dag_vote_qc_fast_path); + assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); } diff --git a/crates/starfish-core/src/consensus/linearizer.rs b/crates/starfish-core/src/consensus/linearizer.rs index 32f8cde2..2315a960 100644 --- a/crates/starfish-core/src/consensus/linearizer.rs +++ b/crates/starfish-core/src/consensus/linearizer.rs @@ -398,6 +398,7 @@ impl Linearizer { } ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed => { self.collect_subdag_acknowledgments(dag_state, leader_block, false) diff --git a/crates/starfish-core/src/consensus/universal_committer.rs b/crates/starfish-core/src/consensus/universal_committer.rs index 7602b00e..eeb0f1ea 100644 --- a/crates/starfish-core/src/consensus/universal_committer.rs +++ b/crates/starfish-core/src/consensus/universal_committer.rs @@ -410,6 +410,7 @@ impl UniversalCommitterBuilder { ConsensusProtocol::Mysticeti | ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::StarfishBls | ConsensusProtocol::MysticetiBls diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index 0a43974a..d2970e8b 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -2,7 +2,7 @@ // Modifications Copyright (c) 2025 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{fmt, mem, sync::Arc}; +use std::{collections::BTreeSet, fmt, mem, sync::Arc}; use ahash::{AHashMap, AHashSet}; use reed_solomon_simd::ReedSolomonEncoder; @@ -39,7 +39,7 @@ use crate::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishFields, Shard, - VerifiedBlock, + StarfishRbcFieldsV3, StarfishRbcReferenceV3, VerifiedBlock, }, }; @@ -54,6 +54,9 @@ pub struct Core { block_manager: BlockManager, pending: Vec, pending_reconstructed_data: AHashMap, + /// Irrevocable local ECHO/READY statements waiting to ride on the next + /// ordinary block in the single-DAG protocol. + pending_starfish_rbc_references: BTreeSet, // For Byzantine node, last_own_block contains a vector of blocks last_own_block: Vec, block_handler: H, @@ -319,6 +322,7 @@ impl Core { store, pending, pending_reconstructed_data: AHashMap::new(), + pending_starfish_rbc_references: BTreeSet::new(), last_own_block: vec![last_own_block], block_handler, authority, @@ -354,6 +358,16 @@ impl Core { &self.signer } + pub(crate) fn add_starfish_rbc_reference(&mut self, reference: StarfishRbcReferenceV3) { + assert!( + self.dag_state + .consensus_protocol + .is_starfish_rbc_single_dag(), + "embedded RBC references require single-DAG Starfish-RBC" + ); + self.pending_starfish_rbc_references.insert(reference); + } + pub(crate) fn get_ml_dsa_44_signer(&self) -> &crate::crypto::MlDsa44Signer { &self.ml_dsa_44_signer } @@ -651,6 +665,7 @@ impl Core { // Dual-DAG protocols: require clean parent quorum before creating a block. if !self.rbc_dag_application_production && protocol.uses_dual_dag() + && !protocol.is_starfish_rbc_single_dag() && clock_round > 1 && !self.dag_state.clean_parent_quorum(clock_round - 1) { @@ -669,6 +684,7 @@ impl Core { // mandatory parent into a proposal. if !self.rbc_dag_application_production && protocol.is_starfish_rbc() + && !protocol.is_starfish_rbc_single_dag() && clock_round > 1 && self .last_own_block @@ -829,6 +845,20 @@ impl Core { } else { None }; + let single_dag_rbc = protocol.is_starfish_rbc_single_dag().then(|| { + let maximum = self.committee.len().saturating_mul(6); + let references: Vec<_> = self + .pending_starfish_rbc_references + .iter() + .copied() + .filter(|evidence| evidence.reference().round <= clock_round) + .take(maximum) + .collect(); + for reference in &references { + self.pending_starfish_rbc_references.remove(reference); + } + StarfishRbcFieldsV3::new(references) + }); // Create and store blocks let mut first_block = None; @@ -849,6 +879,7 @@ impl Core { block_id, aggregate_round_sig, certified_leader, + single_dag_rbc.as_ref(), ); tracing::debug!("Created block {:?}", block_data); if first_block.is_none() { @@ -911,6 +942,10 @@ impl Core { // then be filtered itself, shrinking the usable clean frontier. let (compression_candidates, deferred_dirty_refs): (Vec<_>, Vec<_>) = if self.dag_state.consensus_protocol.is_starfish_rbc() + && !self + .dag_state + .consensus_protocol + .is_starfish_rbc_single_dag() && !self.rbc_dag_application_production { pending_refs.into_iter().partition(|reference| { @@ -927,7 +962,12 @@ impl Core { self.compress_pending_block_references(&compression_candidates, block_round); // Dual-DAG protocols: filter parents to only include clean blocks. - if self.dag_state.consensus_protocol.uses_dual_dag() && !self.rbc_dag_application_production + if self.dag_state.consensus_protocol.uses_dual_dag() + && !self + .dag_state + .consensus_protocol + .is_starfish_rbc_single_dag() + && !self.rbc_dag_application_production { let before = block_references.clone(); block_references.retain(|r| r.round == 0 || self.dag_state.has_clean_vertex(r)); @@ -1113,6 +1153,7 @@ impl Core { block_id_in_round: usize, aggregate_round_sig: Option, certified_leader: Option<(BlockReference, BlsAggregateCertificate)>, + single_dag_rbc: Option<&StarfishRbcFieldsV3>, ) -> Data { let time_ns = timestamp_utc().as_nanos() as u64 + block_id_in_round as u64; let own_previous = *self.last_own_block[block_id_in_round].block.reference(); @@ -1209,7 +1250,20 @@ impl Core { None }; - let mut block = if protocol == ConsensusProtocol::StarfishRbc { + let mut block = if protocol.is_starfish_rbc_single_dag() { + VerifiedBlock::new_starfish_rbc_single_dag( + self.authority, + clock_round, + block_references, + acknowledgment_references.to_vec(), + time_ns, + transactions.to_vec(), + encoded_transactions.clone(), + single_dag_rbc + .cloned() + .expect("single-DAG Starfish-RBC block requires V3 fields"), + ) + } else if protocol == ConsensusProtocol::StarfishRbc { VerifiedBlock::new_starfish_rbc( self.authority, clock_round, @@ -2814,6 +2868,7 @@ mod tests { 0, Some(make_test_round_certificate(&bls_signers, 1)), None, + None, ); let refs = round_2.block_references(); diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index b7ba6a73..65405824 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -81,6 +81,7 @@ enum CoreThreadCommand { ApplySailfishCertificates(Vec, oneshot::Sender<()>), /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), + ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. ApplyStarfishRbcDagFrontier( CommittedFrontierDeltaV1, @@ -272,6 +273,18 @@ impl CoreThread { self.syncer.apply_starfish_rbc_deliveries(delivered_headers); sender.send(()).ok(); } + CoreThreadCommand::ApplyStarfishRbcReference(reference, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_reference"]) + .inc(); + self.syncer.apply_starfish_rbc_reference(reference); + sender.send(()).ok(); + } CoreThreadCommand::ApplyStarfishRbcDagFrontier(delta, sender) => { metrics .core_thread_tasks_total diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index c7f3d7b7..3d85e0bf 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -20,7 +20,7 @@ use crate::{ crypto, types::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockReference, RoundNumber, Shard, - TimestampNs, + StarfishRbcFieldsV3, TimestampNs, }, }; @@ -277,6 +277,55 @@ impl BlockDigest { Self(hasher.finalize().into()) } + /// Canonical identity for a single-DAG Starfish-RBC V3 block. + /// + /// The existing Starfish fields and the typed RBC references share one + /// digest. Authentication therefore binds evidence to the ordinary block + /// author without introducing a carrier or projected-vertex identity. + pub(crate) fn new_starfish_rbc_single_dag_header( + authority: AuthorityIndex, + round: RoundNumber, + block_references: &[BlockReference], + acknowledgment_references: &[BlockReference], + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + rbc: &StarfishRbcFieldsV3, + ) -> Self { + const DOMAIN: &[u8] = b"STARFISH_RBC_SINGLE_DAG_V3"; + + fn hash_reference(hasher: &mut Blake3Hasher, block_ref: &BlockReference) { + hasher.update(&block_ref.authority.to_be_bytes()); + hasher.update(&block_ref.round.to_be_bytes()); + hasher.update(block_ref.digest.as_ref()); + } + + fn hash_references(hasher: &mut Blake3Hasher, references: &[BlockReference]) { + let length = + u32::try_from(references.len()).expect("Starfish-RBC reference count exceeds u32"); + hasher.update(&length.to_be_bytes()); + for reference in references { + hash_reference(hasher, reference); + } + } + + let mut hasher = Blake3Hasher::new(); + hasher.update(DOMAIN); + hasher.update(&authority.to_be_bytes()); + hasher.update(&round.to_be_bytes()); + hash_references(&mut hasher, block_references); + hash_references(&mut hasher, acknowledgment_references); + hasher.update(&meta_creation_time_ns.to_be_bytes()); + hasher.update(transactions_commitment.as_ref()); + let evidence_len = + u32::try_from(rbc.references().len()).expect("Starfish-RBC evidence count exceeds u32"); + hasher.update(&evidence_len.to_be_bytes()); + for evidence in rbc.references() { + hasher.update(&[evidence.kind().tag()]); + hash_reference(&mut hasher, &evidence.reference()); + } + Self(hasher.finalize().into()) + } + pub fn new_without_transactions_with_unprovable( authority: AuthorityIndex, round: RoundNumber, diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index a659b2da..0e57fff8 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -135,6 +135,9 @@ pub enum ConsensusProtocol { /// Plain Starfish ordering over headers certified by the Starfish-RBC /// reliable-broadcast service. StarfishRbc, + /// One-DAG Starfish-RBC: ordinary Starfish blocks carry typed RBC + /// references and retain the same identity from admission through commit. + StarfishRbcSingleDag, StarfishSpeed, StarfishBls, SailfishPlusPlus, @@ -195,6 +198,7 @@ impl ConsensusProtocol { "cordial-miners" => Some(ConsensusProtocol::CordialMiners), "starfish" => Some(ConsensusProtocol::Starfish), "starfish-rbc" => Some(ConsensusProtocol::StarfishRbc), + "starfish-rbc-single-dag" => Some(ConsensusProtocol::StarfishRbcSingleDag), "starfish-bls" | "starfish-l" => Some(ConsensusProtocol::StarfishBls), "starfish-speed" | "starfish-s" => Some(ConsensusProtocol::StarfishSpeed), "sailfish++" | "sailfish-pp" => Some(ConsensusProtocol::SailfishPlusPlus), @@ -212,6 +216,7 @@ impl ConsensusProtocol { self, ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishBls | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed @@ -223,7 +228,14 @@ impl ConsensusProtocol { } pub fn is_starfish_rbc(self) -> bool { - matches!(self, ConsensusProtocol::StarfishRbc) + matches!( + self, + ConsensusProtocol::StarfishRbc | ConsensusProtocol::StarfishRbcSingleDag + ) + } + + pub fn is_starfish_rbc_single_dag(self) -> bool { + matches!(self, ConsensusProtocol::StarfishRbcSingleDag) } pub fn is_bluestreak(self) -> bool { @@ -267,6 +279,7 @@ impl ConsensusProtocol { self, ConsensusProtocol::SailfishPlusPlus | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::Bluestreak | ConsensusProtocol::StarfishBls | ConsensusProtocol::MysticetiBls @@ -293,6 +306,7 @@ impl ConsensusProtocol { ConsensusProtocol::CordialMiners => DisseminationMode::PushCausal, ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed | ConsensusProtocol::SparseStarfishSpeed => DisseminationMode::PushUseful, } @@ -945,6 +959,9 @@ impl DagState { ConsensusProtocol::Mysticeti => tracing::info!("Starting Mysticeti protocol"), ConsensusProtocol::Starfish => tracing::info!("Starting Starfish protocol"), ConsensusProtocol::StarfishRbc => tracing::info!("Starting Starfish-RBC protocol"), + ConsensusProtocol::StarfishRbcSingleDag => { + tracing::info!("Starting single-DAG Starfish-RBC protocol") + } ConsensusProtocol::StarfishBls => tracing::info!("Starting Starfish-BLS protocol"), ConsensusProtocol::StarfishSpeed => tracing::info!("Starting Starfish-Speed protocol"), ConsensusProtocol::CordialMiners => tracing::info!("Starting Cordial Miners protocol"), @@ -1016,7 +1033,9 @@ impl DagState { /// round `r - 1`. BLS protocols additionally gate on the highest BLS /// round certificate + 1. Other protocols use the raw threshold clock. pub fn proposal_round(&self) -> RoundNumber { - if !self.consensus_protocol.uses_dual_dag() { + if !self.consensus_protocol.uses_dual_dag() + || self.consensus_protocol.is_starfish_rbc_single_dag() + { return self.threshold_clock_round(); } @@ -1962,7 +1981,9 @@ impl DagState { let inner = self.dag_state_inner.read(); let leader_round = quorum_round - 1; let mut blocks = inner.get_blocks_by_round(leader_round); - if self.consensus_protocol.is_starfish_rbc() { + if self.consensus_protocol.is_starfish_rbc() + && !self.consensus_protocol.is_starfish_rbc_single_dag() + { blocks.retain(|block| { inner.clean_vertices[block.authority() as usize].contains(block.reference()) }); @@ -2048,6 +2069,7 @@ impl DagState { } if self.consensus_protocol.uses_dual_dag() + && !self.consensus_protocol.is_starfish_rbc_single_dag() && quorum_round > 1 && !self.clean_parent_quorum(quorum_round - 1) { @@ -3775,8 +3797,8 @@ mod tests { types::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthentication, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, - Encoder, ProvableShard, RoundNumber, SailfishFields, SailfishNoVoteCert, Transaction, - TransactionData, VerifiedBlock, + Encoder, ProvableShard, RoundNumber, SailfishFields, SailfishNoVoteCert, + StarfishRbcFieldsV3, Transaction, TransactionData, VerifiedBlock, }, }; @@ -3980,6 +4002,25 @@ mod tests { Data::new(block) } + fn make_starfish_rbc_single_dag_block( + authority: AuthorityIndex, + round: RoundNumber, + parents: Vec, + ) -> Data { + let mut block = VerifiedBlock::new_starfish_rbc_single_dag( + authority, + round, + parents, + Vec::new(), + round as u64, + Vec::new(), + None, + StarfishRbcFieldsV3::default(), + ); + block.preserialize(); + Data::new(block) + } + #[test] fn acknowledgments_are_only_enabled_for_starfish_variants() { assert!(!ConsensusProtocol::Mysticeti.supports_acknowledgments()); @@ -4330,6 +4371,37 @@ mod tests { ); } + #[test] + fn starfish_rbc_single_dag_dirty_quorum_advances_production_not_clean_consensus() { + let dag_state = open_test_dag_state_for("starfish-rbc-single-dag", 0); + let committee = Committee::new_for_benchmarks(4); + let genesis: Vec<_> = (0..4) + .map(|auth| BlockReference::new_test(auth, 0)) + .collect(); + let round_one: Vec<_> = (0..4) + .map(|authority| make_starfish_rbc_single_dag_block(authority, 1, genesis.clone())) + .collect(); + let round_one_refs: Vec<_> = round_one.iter().map(|block| *block.reference()).collect(); + + dag_state.insert_general_blocks(round_one, DataSource::BlockBundleStreaming); + + assert_eq!(dag_state.threshold_clock_round(), 2); + assert_eq!(dag_state.proposal_round(), 2); + assert!(!dag_state.clean_parent_quorum(1)); + assert!( + round_one_refs + .iter() + .all(|reference| !dag_state.has_clean_vertex(reference)) + ); + assert!(dag_state.is_ready_for_new_block( + 2, + &[committee.elect_leader(1)], + false, + 0, + committee.as_ref(), + )); + } + #[test] fn starfish_rbc_acknowledgment_waits_for_both_cleanliness_and_data() { let dag_state = open_test_dag_state_for("starfish-rbc", 0); @@ -5411,6 +5483,11 @@ mod tests { ); assert!(ConsensusProtocol::StarfishRbc.is_starfish_rbc()); assert!(ConsensusProtocol::StarfishRbc.uses_dual_dag()); + assert!(ConsensusProtocol::StarfishRbcSingleDag.is_starfish_rbc_single_dag()); + assert_eq!( + ConsensusProtocol::StarfishRbcSingleDag.default_dissemination_mode(), + DisseminationMode::PushUseful + ); assert_eq!( ConsensusProtocol::StarfishSpeed.default_dissemination_mode(), DisseminationMode::PushUseful @@ -5433,6 +5510,10 @@ mod tests { ("cordial-miners", ConsensusProtocol::CordialMiners), ("starfish", ConsensusProtocol::Starfish), ("starfish-rbc", ConsensusProtocol::StarfishRbc), + ( + "starfish-rbc-single-dag", + ConsensusProtocol::StarfishRbcSingleDag, + ), ("starfish-speed", ConsensusProtocol::StarfishSpeed), ("starfish-bls", ConsensusProtocol::StarfishBls), ("sailfish-pp", ConsensusProtocol::SailfishPlusPlus), diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 9ec6ffe6..9f4ec5e7 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -72,7 +72,7 @@ use crate::{ RbcInitialAuthenticator, RbcPhaseAuthorityV1, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service_with_phase_authority, }, - syncer::{CommitObserver, Syncer, SyncerSignals}, + syncer::{CommitObserver, STARFISH_RBC_SINGLE_DAG_ROUND_INTERVAL, Syncer, SyncerSignals}, types::{ AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, @@ -1963,6 +1963,7 @@ impl ConnectionHandler { @@ -3202,7 +3203,14 @@ impl NetworkSyncer initial_authenticator, dag_state.highest_round(), STARFISH_RBC_HEADER_RETRY_INTERVAL, - RbcPhaseAuthorityV1::Direct, + if dag_state.consensus_protocol.is_starfish_rbc_single_dag() { + RbcPhaseAuthorityV1::EmbeddedSingleDag { + echo_qc_fast_path: node_parameters + .starfish_rbc_single_dag_echo_qc_fast_path, + } + } else { + RbcPhaseAuthorityV1::Direct + }, ) .expect("validated Starfish-RBC configuration must start its service"); (Some(service), Some(events), Some(task)) @@ -3554,6 +3562,12 @@ impl NetworkSyncer .apply_starfish_rbc_deliveries(vec![header]) .await; } + RbcServiceEvent::ReferenceReady(reference) => { + event_inner + .syncer + .apply_starfish_rbc_reference(reference) + .await; + } RbcServiceEvent::Rejected { peer, error } => { tracing::warn!( "Rejected Starfish-RBC input from {:?}: {}", @@ -5006,7 +5020,15 @@ impl NetworkSyncer for round in armed_round + 1..=current_round { let timer_inner = inner.clone(); Handle::current().spawn(async move { - let leader_timeout = timer_inner.leader_timeout; + let leader_timeout = if timer_inner + .dag_state + .consensus_protocol + .is_starfish_rbc_single_dag() + { + STARFISH_RBC_SINGLE_DAG_ROUND_INTERVAL + } else { + timer_inner.leader_timeout + }; select! { _sleep = sleep(leader_timeout) => { tracing::debug!("Timeout for proposal round {round}"); diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index b6ace62c..4096a982 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -21,8 +21,8 @@ use crate::{ types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - TimestampNs, TransactionData, VerifiedBlock, compress_acknowledgments, - expand_acknowledgments, + StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, + TransactionData, VerifiedBlock, compress_acknowledgments, expand_acknowledgments, }, }; @@ -157,6 +157,8 @@ pub struct RbcCanonicalHeader { acknowledgments: RbcAckFields, meta_creation_time_ns: TimestampNs, transactions_commitment: TransactionsCommitment, + #[serde(default)] + starfish_rbc_v3: Option, } impl RbcCanonicalHeader { @@ -167,6 +169,46 @@ impl RbcCanonicalHeader { acknowledgment_references: Vec, meta_creation_time_ns: TimestampNs, transactions_commitment: TransactionsCommitment, + ) -> Result { + Self::try_new_with_fields( + authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions_commitment, + None, + ) + } + + pub(crate) fn try_new_single_dag( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + starfish_rbc_v3: StarfishRbcFieldsV3, + ) -> Result { + Self::try_new_with_fields( + authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions_commitment, + Some(starfish_rbc_v3), + ) + } + + fn try_new_with_fields( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + starfish_rbc_v3: Option, ) -> Result { for (field, count) in [ ("parent", block_references.len()), @@ -196,10 +238,17 @@ impl RbcCanonicalHeader { let acknowledgments = RbcAckFields::from_logical(&block_references, &acknowledgment_references); let logical_acknowledgments = acknowledgments.logical(&block_references); - let reference = BlockReference { - authority, - round, - digest: BlockDigest::new_starfish_rbc_header( + let digest = match starfish_rbc_v3.as_ref() { + Some(rbc) => BlockDigest::new_starfish_rbc_single_dag_header( + authority, + round, + &block_references, + &logical_acknowledgments, + meta_creation_time_ns, + transactions_commitment, + rbc, + ), + None => BlockDigest::new_starfish_rbc_header( authority, round, &block_references, @@ -208,12 +257,18 @@ impl RbcCanonicalHeader { transactions_commitment, ), }; + let reference = BlockReference { + authority, + round, + digest, + }; let header = Self { reference, block_references, acknowledgments, meta_creation_time_ns, transactions_commitment, + starfish_rbc_v3, }; if header.encoded_content_size(logical_acknowledgments.len())? > MAX_RBC_HEADER_CONTENT_SIZE { @@ -268,6 +323,14 @@ impl RbcCanonicalHeader { } let reference_count = parent_count .checked_add(logical_acknowledgment_count) + .and_then(|count| { + count.checked_add( + header + .starfish_rbc_v3 + .as_ref() + .map_or(0, |rbc| rbc.references().len()), + ) + }) .ok_or(RbcError::HeaderContentTooLarge)?; let encoded_size = RBC_BLOCK_REFERENCE_SIZE .checked_mul(reference_count) @@ -285,6 +348,7 @@ impl RbcCanonicalHeader { }, meta_creation_time_ns: header.meta_creation_time_ns, transactions_commitment, + starfish_rbc_v3: header.starfish_rbc_v3.clone(), }) } @@ -315,6 +379,10 @@ impl RbcCanonicalHeader { self.transactions_commitment } + pub fn starfish_rbc_v3(&self) -> Option<&StarfishRbcFieldsV3> { + self.starfish_rbc_v3.as_ref() + } + /// Validate canonical header content against an already validated static /// committee without deriving the committee identifier. /// @@ -384,14 +452,32 @@ impl RbcCanonicalHeader { } } - let expected_digest = BlockDigest::new_starfish_rbc_header( - block_ref.authority, - block_ref.round, - &self.block_references, - &acknowledgments, - self.meta_creation_time_ns, - self.transactions_commitment, - ); + if self + .starfish_rbc_v3 + .as_ref() + .is_some_and(|rbc| !rbc.validate_for_block(committee, block_ref.round)) + { + return Err(RbcError::InvalidSingleDagEvidence); + } + let expected_digest = match self.starfish_rbc_v3.as_ref() { + Some(rbc) => BlockDigest::new_starfish_rbc_single_dag_header( + block_ref.authority, + block_ref.round, + &self.block_references, + &acknowledgments, + self.meta_creation_time_ns, + self.transactions_commitment, + rbc, + ), + None => BlockDigest::new_starfish_rbc_header( + block_ref.authority, + block_ref.round, + &self.block_references, + &acknowledgments, + self.meta_creation_time_ns, + self.transactions_commitment, + ), + }; if expected_digest != block_ref.digest { return Err(RbcError::HeaderDigestMismatch { expected: expected_digest, @@ -417,6 +503,7 @@ impl RbcCanonicalHeader { bls: None, sailfish: None, unprovable_certificate: None, + starfish_rbc_v3: self.starfish_rbc_v3.clone(), serialized: None, }, None, @@ -428,6 +515,13 @@ impl RbcCanonicalHeader { .block_references .len() .checked_add(acknowledgment_count) + .and_then(|count| { + count.checked_add( + self.starfish_rbc_v3 + .as_ref() + .map_or(0, |rbc| rbc.references().len()), + ) + }) .ok_or(RbcError::HeaderContentTooLarge)?; RBC_BLOCK_REFERENCE_SIZE .checked_mul(reference_count) @@ -863,6 +957,7 @@ pub(crate) enum RbcError { AcknowledgmentFromFuture(BlockReference), DuplicateAcknowledgment(BlockReference), InvalidThresholdClock, + InvalidSingleDagEvidence, HeaderDigestMismatch { expected: BlockDigest, actual: BlockDigest, @@ -1001,6 +1096,9 @@ impl fmt::Display for RbcError { Self::InvalidThresholdClock => { f.write_str("Starfish-RBC header does not reference previous-round quorum stake") } + Self::InvalidSingleDagEvidence => { + f.write_str("Starfish-RBC V3 block carries non-canonical reference evidence") + } Self::HeaderDigestMismatch { expected, actual } => write!( f, "Starfish-RBC header digest mismatch: expected {expected}, got {actual}" @@ -1168,10 +1266,16 @@ pub(crate) struct StarfishRbcKernel { mac_keys: Arc>, local_round: RoundNumber, minimum_new_slot_round: RoundNumber, + /// Testbed-only optimistic path. A quorum of locked ECHOs proves a unique + /// value, but without a portable proof it does not prove that every honest + /// node can assemble the same quorum under selective Byzantine + /// withholding. Keep disabled for the asynchronous RBC contract. + echo_qc_fast_path: bool, slots: BTreeMap>, } impl StarfishRbcKernel { + #[allow(dead_code)] pub(crate) fn new( committee: Arc, own_authority: AuthorityIndex, @@ -1179,6 +1283,26 @@ impl StarfishRbcKernel { initial_authentication: BlockAuthenticationScheme, mac_keys: Arc>, local_round: RoundNumber, + ) -> Result { + Self::new_with_echo_qc_fast_path( + committee, + own_authority, + protocol_instance, + initial_authentication, + mac_keys, + local_round, + false, + ) + } + + pub(crate) fn new_with_echo_qc_fast_path( + committee: Arc, + own_authority: AuthorityIndex, + protocol_instance: RbcProtocolInstanceId, + initial_authentication: BlockAuthenticationScheme, + mac_keys: Arc>, + local_round: RoundNumber, + echo_qc_fast_path: bool, ) -> Result { let context = RbcContext::new(protocol_instance, &committee, initial_authentication)?; if !committee.known_authority(own_authority) { @@ -1197,6 +1321,7 @@ impl StarfishRbcKernel { mac_keys, local_round, minimum_new_slot_round: 1, + echo_qc_fast_path, slots: BTreeMap::new(), }) } @@ -1356,14 +1481,44 @@ impl StarfishRbcKernel { meta_creation_time_ns: TimestampNs, transactions_commitment: TransactionsCommitment, ) -> Result { - let canonical = RbcCanonicalHeader::try_new( - self.own_authority, + self.start_local_initial_header_with_fields( round, block_references, acknowledgment_references, meta_creation_time_ns, transactions_commitment, - )?; + None, + ) + } + + pub(crate) fn start_local_initial_header_with_fields( + &mut self, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions_commitment: TransactionsCommitment, + starfish_rbc_v3: Option, + ) -> Result { + let canonical = match starfish_rbc_v3 { + Some(rbc) => RbcCanonicalHeader::try_new_single_dag( + self.own_authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions_commitment, + rbc, + ), + None => RbcCanonicalHeader::try_new( + self.own_authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions_commitment, + ), + }?; let pinned = self.validate_header_content(canonical)?; let eligible = self.local_initial_header(pinned.clone())?; let effects = self.accept_initial_header(eligible)?; @@ -1520,6 +1675,43 @@ impl StarfishRbcKernel { Ok(self.drive(message.block_ref)) } + /// Apply a statement authenticated by the ordinary V3 block that carries + /// it. The carrying block author is the RBC sender; no standalone phase + /// MAC or second network message exists in this path. + pub(crate) fn handle_embedded_reference( + &mut self, + authenticated_sender: AuthorityIndex, + evidence: StarfishRbcReferenceV3, + ) -> Result, RbcError> { + if !self.committee.known_authority(authenticated_sender) { + return Err(RbcError::UnknownAuthority(authenticated_sender)); + } + let block_ref = evidence.reference(); + self.validate_block_ref(&block_ref)?; + let phase = match evidence.kind() { + StarfishRbcReferenceKindV3::Echo => RbcPhase::Echo, + StarfishRbcReferenceKindV3::Ready => RbcPhase::Ready, + }; + let committee = Arc::clone(&self.committee); + let slot = self.slot_mut(block_ref); + if !slot.record_phase_sender(phase, authenticated_sender, block_ref) { + return Ok(Vec::new()); + } + let candidate = slot + .candidates + .entry(block_ref) + .or_insert_with(CandidateState::new); + match phase { + RbcPhase::Echo => { + candidate.echoes.add(authenticated_sender, &committee); + } + RbcPhase::Ready => { + candidate.readies.add(authenticated_sender, &committee); + } + } + Ok(self.drive(block_ref)) + } + /// Materialize one recipient-specific message for an untagged multicast /// effect. The network adapter calls this once per non-local recipient. pub(crate) fn make_phase_message( @@ -1840,6 +2032,7 @@ impl StarfishRbcKernel { fn drive(&mut self, block_ref: BlockReference) -> Vec { let validity_threshold = self.committee.validity_threshold(); let quorum_threshold = self.committee.quorum_threshold(); + let echo_qc_fast_path = self.echo_qc_fast_path; let mut effects = Vec::new(); loop { @@ -1866,7 +2059,8 @@ impl StarfishRbcKernel { ProgressAction::SendReady } else if candidate.header.is_some() && can_deliver - && candidate.ready_quorum_observed + && (candidate.ready_quorum_observed + || (echo_qc_fast_path && candidate.echo_quorum_observed)) { ProgressAction::Deliver } else { @@ -2051,6 +2245,7 @@ mod tests { }, meta_creation_time_ns: 0, transactions_commitment: TransactionsCommitment::default(), + starfish_rbc_v3: None, }), committee_id: context.committee_id, } @@ -2068,6 +2263,102 @@ mod tests { ) } + #[test] + fn embedded_block_references_drive_rbc_without_phase_messages() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = mac_keyrings_for_test(committee.len()); + let mut receiver = StarfishRbcKernel::new( + committee.clone(), + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + 1, + ) + .unwrap(); + let target = block(3, 1, 0x71); + receiver + .note_header_available(pinned_header_for_context(receiver.context, target)) + .unwrap(); + receiver.authorize_echo(target).unwrap(); + + let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); + assert!( + receiver + .handle_embedded_reference(1, echo) + .unwrap() + .is_empty() + ); + let effects = receiver.handle_embedded_reference(2, echo).unwrap(); + assert!(effects.iter().any(|effect| matches!( + effect, + RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + } if *block_ref == target + ))); + assert!( + !effects + .iter() + .any(|effect| matches!(effect, RbcEffect::Deliver(_))) + ); + + let ready = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Ready, target); + assert!( + receiver + .handle_embedded_reference(1, ready) + .unwrap() + .is_empty() + ); + let effects = receiver.handle_embedded_reference(2, ready).unwrap(); + assert!(effects.iter().any(|effect| matches!( + effect, + RbcEffect::Deliver(header) if header.reference() == target + ))); + } + + #[test] + fn flagged_echo_qc_fast_path_delivers_unique_header_without_ready_quorum() { + let committee = Committee::new_for_benchmarks(4); + let keyrings = mac_keyrings_for_test(committee.len()); + let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( + committee, + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + 1, + true, + ) + .unwrap(); + let target = block(3, 1, 0x72); + receiver + .note_header_available(pinned_header_for_context(receiver.context, target)) + .unwrap(); + receiver.authorize_echo(target).unwrap(); + + let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); + assert!( + receiver + .handle_embedded_reference(1, echo) + .unwrap() + .is_empty() + ); + let effects = receiver.handle_embedded_reference(2, echo).unwrap(); + + assert!(effects.iter().any(|effect| matches!( + effect, + RbcEffect::MulticastPhase { + phase: RbcPhase::Ready, + block_ref, + } if *block_ref == target + ))); + assert!(effects.iter().any(|effect| matches!( + effect, + RbcEffect::Deliver(header) if header.reference() == target + ))); + } + fn valid_canonical_header( authority: AuthorityIndex, round: RoundNumber, @@ -2108,6 +2399,7 @@ mod tests { bls: None, sailfish: None, unprovable_certificate: None, + starfish_rbc_v3: None, serialized: None, } } diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 721be73a..8b08038a 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -34,7 +34,8 @@ use crate::{ }, types::{ AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - RoundNumber, TimestampNs, TransactionData, + RoundNumber, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, + TimestampNs, TransactionData, }, }; @@ -69,6 +70,7 @@ pub(crate) struct RbcLocalHeader { pub acknowledgment_references: Vec, pub meta_creation_time_ns: TimestampNs, pub transactions_commitment: TransactionsCommitment, + pub starfish_rbc_v3: Option, } impl RbcLocalHeader { @@ -79,6 +81,7 @@ impl RbcLocalHeader { acknowledgment_references: header.acknowledgment_references(), meta_creation_time_ns: header.meta_creation_time_ns(), transactions_commitment: header.transactions_commitment(), + starfish_rbc_v3: header.starfish_rbc_v3().cloned(), } } } @@ -154,6 +157,9 @@ pub(crate) enum RbcServiceEvent { transaction_data: Arc, }, Delivered(PinnedRbcHeader), + /// An irrevocable local phase statement waiting to be embedded in the + /// next ordinary Starfish block. + ReferenceReady(StarfishRbcReferenceV3), Rejected { peer: Option, error: RbcServiceError, @@ -202,6 +208,7 @@ pub(crate) struct RbcServiceHandle { pub(crate) enum RbcPhaseAuthorityV1 { Direct, EmbeddedCarrierDag, + EmbeddedSingleDag { echo_qc_fast_path: bool }, } impl RbcServiceHandle { @@ -381,13 +388,18 @@ pub(crate) fn start_starfish_rbc_service_with_phase_authority( } validate_local_authenticator(&committee, own_authority, &initial_authenticator)?; - let kernel = StarfishRbcKernel::new( + let echo_qc_fast_path = match phase_authority { + RbcPhaseAuthorityV1::EmbeddedSingleDag { echo_qc_fast_path } => echo_qc_fast_path, + RbcPhaseAuthorityV1::Direct | RbcPhaseAuthorityV1::EmbeddedCarrierDag => false, + }; + let kernel = StarfishRbcKernel::new_with_echo_qc_fast_path( committee.clone(), own_authority, protocol_instance, initial_authentication, mac_keys, local_round, + echo_qc_fast_path, )?; let (message_tx, message_rx) = mpsc::unbounded_channel(); let (event_tx, event_rx) = mpsc::unbounded_channel(); @@ -529,14 +541,25 @@ impl RbcServiceState { transaction_data: Option, ) -> Result { self.kernel.advance_local_round(header.round)?; - let local = self.kernel.start_local_initial_header( - header.round, - header.block_references, - header.acknowledgment_references, - header.meta_creation_time_ns, - header.transactions_commitment, - )?; + let local = match header.starfish_rbc_v3 { + Some(rbc) => self.kernel.start_local_initial_header_with_fields( + header.round, + header.block_references, + header.acknowledgment_references, + header.meta_creation_time_ns, + header.transactions_commitment, + Some(rbc), + ), + None => self.kernel.start_local_initial_header( + header.round, + header.block_references, + header.acknowledgment_references, + header.meta_creation_time_ns, + header.transactions_commitment, + ), + }?; let canonical = local.header().clone(); + let embedded_references = canonical.starfish_rbc_v3().cloned(); let transaction_data = transaction_data.map(Arc::new); let proposals = self.make_initial_proposals(&local, transaction_data); let (pinned, effects) = local.into_parts(); @@ -548,6 +571,7 @@ impl RbcServiceState { self.send_network(recipient, NetworkMessage::RbcInitial(proposal)); } self.process_effects(effects); + self.process_embedded_references(self.own_authority, embedded_references); Ok(canonical) } @@ -631,6 +655,7 @@ impl RbcServiceState { fn accept_direct_initial(&mut self, peer: AuthorityIndex, proposal: RbcHeaderProposal) { let (header, proof, transaction_data) = proposal.into_parts(); let block_ref = header.reference(); + let embedded_references = header.starfish_rbc_v3().cloned(); match self .kernel .accept_direct_initial_header(peer, header, &proof) @@ -639,6 +664,7 @@ impl RbcServiceState { let pinned = self.finish_header_staging(block_ref, Some(peer)); self.notify_transaction_payload(peer, pinned, transaction_data); self.process_effects(effects); + self.process_embedded_references(peer, embedded_references); } Ok(RbcInitialHeaderOutcome::StagedUnauthenticated { effects, error }) => { let pinned = self.finish_header_staging(block_ref, Some(peer)); @@ -753,6 +779,19 @@ impl RbcServiceState { if self.phase_authority == RbcPhaseAuthorityV1::EmbeddedCarrierDag { continue; } + if matches!( + self.phase_authority, + RbcPhaseAuthorityV1::EmbeddedSingleDag { .. } + ) { + let kind = match phase { + RbcPhase::Echo => StarfishRbcReferenceKindV3::Echo, + RbcPhase::Ready => StarfishRbcReferenceKindV3::Ready, + }; + let _ = self.events.send(RbcServiceEvent::ReferenceReady( + StarfishRbcReferenceV3::new(kind, block_ref), + )); + continue; + } self.retained_phases.insert((block_ref, phase)); let recipients: Vec<_> = self .committee @@ -782,6 +821,32 @@ impl RbcServiceState { } } + fn process_embedded_references( + &mut self, + sender: AuthorityIndex, + references: Option, + ) { + if !matches!( + self.phase_authority, + RbcPhaseAuthorityV1::EmbeddedSingleDag { .. } + ) { + return; + } + let Some(references) = references else { + self.reject( + Some(sender), + RbcServiceError::Kernel(RbcError::InvalidSingleDagEvidence), + ); + return; + }; + for evidence in references.references() { + match self.kernel.handle_embedded_reference(sender, *evidence) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(sender), error.into()), + } + } + } + fn note_pending_fetch(&mut self, block_ref: BlockReference, holders: AuthoritySet) { self.pending_fetches .entry(block_ref) @@ -934,6 +999,7 @@ mod tests { acknowledgment_references: Vec::new(), meta_creation_time_ns: 17, transactions_commitment: TransactionsCommitment::default(), + starfish_rbc_v3: None, } } @@ -991,6 +1057,29 @@ mod tests { .unwrap() } + fn start_single_dag_service() -> ( + RbcServiceHandle, + mpsc::UnboundedReceiver, + JoinHandle<()>, + ) { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + start_starfish_rbc_service_with_phase_authority( + committee, + 0, + instance(), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + RbcInitialAuthenticator::Mac, + 1, + Duration::from_secs(3_600), + RbcPhaseAuthorityV1::EmbeddedSingleDag { + echo_qc_fast_path: false, + }, + ) + .unwrap() + } + async fn next_event(events: &mut mpsc::UnboundedReceiver) -> RbcServiceEvent { tokio::time::timeout(Duration::from_secs(2), events.recv()) .await @@ -1096,6 +1185,44 @@ mod tests { task.await.unwrap(); } + #[tokio::test] + async fn single_dag_phase_authority_emits_typed_reference_not_phase_message() { + let (handle, mut events, task) = start_single_dag_service(); + let mut header = local_header(1, 4); + header.starfish_rbc_v3 = Some(StarfishRbcFieldsV3::default()); + let canonical = handle.start_local_header(header).await.unwrap(); + let mut staged = false; + let mut initials = 0; + let mut references = 0; + for _ in 0..5 { + match next_event(&mut events).await { + RbcServiceEvent::HeaderStaged(header) => { + assert_eq!(header.reference(), canonical.reference()); + staged = true; + } + RbcServiceEvent::Network { + message: NetworkMessage::RbcInitial(_), + .. + } => initials += 1, + RbcServiceEvent::ReferenceReady(reference) => { + assert_eq!(reference.kind(), StarfishRbcReferenceKindV3::Echo); + assert_eq!(reference.reference(), canonical.reference()); + references += 1; + } + RbcServiceEvent::Network { + message: NetworkMessage::RbcPhase(_), + .. + } => panic!("single-DAG mode emitted a standalone phase message"), + event => panic!("unexpected single-DAG startup event: {event:?}"), + } + } + assert!(staged); + assert_eq!(initials, 3); + assert_eq!(references, 1); + drop(handle); + task.await.unwrap(); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blocking_local_start_waits_for_kernel_selection_and_event_enqueue() { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 9b7d4a84..ae1f9523 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -2,7 +2,11 @@ // Modifications Copyright (c) 2025 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{collections::BTreeSet, sync::Arc, time::Instant}; +use std::{ + collections::BTreeSet, + sync::Arc, + time::{Duration, Instant}, +}; use ahash::AHashSet; @@ -41,6 +45,13 @@ pub enum BlockCreationReason { PostCommit, } +/// Testbed pacing for the V3 single-DAG protocol. Authenticated quorum may +/// advance the production clock immediately, but an honest authority fixes at +/// most one ordinary block per interval. This prevents a zero-latency quorum +/// from becoming a self-sustaining empty-block loop while leaving the existing +/// leader timeout and all consensus thresholds unchanged. +pub(crate) const STARFISH_RBC_SINGLE_DAG_ROUND_INTERVAL: Duration = Duration::from_millis(50); + impl BlockCreationReason { pub fn as_str(self) -> &'static str { match self { @@ -60,6 +71,7 @@ pub struct Syncer { forced_block_rounds: BTreeSet, proposal_wait_started_at: Option, proposal_wait_round: Option, + single_dag_last_proposal_at: Option, signals: S, commit_observer: C, pub(crate) connected_authorities: AHashSet, @@ -129,6 +141,7 @@ impl Syncer { forced_block_rounds: BTreeSet::new(), proposal_wait_started_at: None, proposal_wait_round: None, + single_dag_last_proposal_at: None, signals, commit_observer, connected_authorities: AHashSet::with_capacity(committee_size), @@ -386,6 +399,17 @@ impl Syncer { } } + /// Queue one locally locked RBC statement for the next ordinary + /// single-DAG block. The statement changes no delivery state until peers + /// authenticate the carrying block. + pub fn apply_starfish_rbc_reference( + &mut self, + reference: crate::types::StarfishRbcReferenceV3, + ) { + self.core.add_starfish_rbc_reference(reference); + self.try_new_block(BlockCreationReason::CertificateEvent); + } + /// Sequence one exact deterministic carrier-frontier delta. In M7 this is /// the sole application-ordering authority; the legacy Starfish committer /// remains disabled in this mode. @@ -562,6 +586,18 @@ impl Syncer { } else { reason }; + if self + .core + .dag_state() + .consensus_protocol + .is_starfish_rbc_single_dag() + && !matches!(effective_reason, BlockCreationReason::ForceTimeout) + && self.single_dag_last_proposal_at.is_some_and(|created_at| { + created_at.elapsed() < STARFISH_RBC_SINGLE_DAG_ROUND_INTERVAL + }) + { + return false; + } self.create_new_block(effective_reason) } @@ -572,6 +608,14 @@ impl Syncer { tracing::debug!("Attempt to create new block in syncer after one trigger"); let previous_rounds = self.capture_rounds(); if let Some(ref block) = self.core.try_new_block(reason.as_str()) { + if self + .core + .dag_state() + .consensus_protocol + .is_starfish_rbc_single_dag() + { + self.single_dag_last_proposal_at = Some(Instant::now()); + } if self.core.dag_state().consensus_protocol.is_starfish_rbc() { let canonical = RbcCanonicalHeader::from_block_header(block.header()) .expect("locally built Starfish-RBC block must have canonical header content"); diff --git a/crates/starfish-core/src/threshold_clock.rs b/crates/starfish-core/src/threshold_clock.rs index 7ba883b0..8a9d8c79 100644 --- a/crates/starfish-core/src/threshold_clock.rs +++ b/crates/starfish-core/src/threshold_clock.rs @@ -113,6 +113,7 @@ mod tests { bls: None, sailfish: None, unprovable_certificate: None, + starfish_rbc_v3: None, serialized: None, } } diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 9620da6b..6edca2ff 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -112,6 +112,89 @@ pub struct AckFields { pub(crate) extra_references: Vec, } +/// The reliable-broadcast statement carried by an ordinary Starfish block in +/// the single-DAG protocol. The carrying block's author authentication also +/// authenticates these references; they deliberately have no second block or +/// carrier identity. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub enum StarfishRbcReferenceKindV3 { + Echo, + Ready, +} + +impl StarfishRbcReferenceKindV3 { + pub(crate) fn tag(self) -> u8 { + match self { + Self::Echo => 0x01, + Self::Ready => 0x02, + } + } +} + +/// One typed RBC reference embedded in a normal Starfish block. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub struct StarfishRbcReferenceV3 { + kind: StarfishRbcReferenceKindV3, + reference: BlockReference, +} + +impl StarfishRbcReferenceV3 { + pub fn new(kind: StarfishRbcReferenceKindV3, reference: BlockReference) -> Self { + Self { kind, reference } + } + + pub fn kind(self) -> StarfishRbcReferenceKindV3 { + self.kind + } + + pub fn reference(self) -> BlockReference { + self.reference + } +} + +/// Versioned single-DAG Starfish-RBC extension. +/// +/// `Some(empty)` is meaningful: it identifies a V3 block even when no RBC +/// statement is ready in that block. References are canonicalized so arrival +/// order cannot create multiple block identities for the same evidence set. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct StarfishRbcFieldsV3 { + references: Vec, +} + +impl StarfishRbcFieldsV3 { + pub fn new(mut references: Vec) -> Self { + references.sort_unstable(); + references.dedup(); + Self { references } + } + + pub fn references(&self) -> &[StarfishRbcReferenceV3] { + &self.references + } + + pub(crate) fn validate_for_block( + &self, + committee: &Committee, + block_round: RoundNumber, + ) -> bool { + if self.references.len() > committee.len().saturating_mul(6) { + return false; + } + if self.references.windows(2).any(|pair| pair[0] >= pair[1]) { + return false; + } + let mut statements = AHashSet::new(); + self.references.iter().all(|evidence| { + let reference = evidence.reference(); + reference.round > 0 + && reference.round <= block_round + && committee.known_authority(reference.authority) + && statements.insert((evidence.kind(), reference.authority, reference.round)) + }) + } +} + /// BLS certificate data (StarfishBls only). /// /// `certified_leader` pairs the leader ref with an aggregate certificate once @@ -416,6 +499,10 @@ pub struct BlockHeader { /// Some(empty)`; `false` (standard) means 2f+1 voters reference the /// leader but the strong-vote quorum is mixed. pub(crate) unprovable_certificate: Option<(BlockReference, bool)>, + /// Single-DAG Starfish-RBC evidence. `None` is the frozen direct-RBC V1 + /// header; `Some` selects the V3 identity and codec domain. + #[serde(default)] + pub(crate) starfish_rbc_v3: Option, // -- Cache (not serialized) ----------------------------------------------- /// Cached bincode-serialized bytes. Populated by `preserialize()` off the @@ -468,6 +555,10 @@ impl BlockHeader { ) } + pub fn starfish_rbc_v3(&self) -> Option<&StarfishRbcFieldsV3> { + self.starfish_rbc_v3.as_ref() + } + pub fn authority(&self) -> AuthorityIndex { self.reference.authority } @@ -858,6 +949,55 @@ impl VerifiedBlock { meta_creation_time_ns: TimestampNs, transactions: Vec, encoded_transactions: Option>, + ) -> Self { + Self::new_starfish_rbc_with_fields( + authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions, + encoded_transactions, + None, + ) + } + + /// Construct one ordinary Starfish block carrying V3 RBC reference + /// evidence. This is the single-DAG path: the resulting `BlockReference` + /// identifies both the consensus vertex and the RBC proposal. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new_starfish_rbc_single_dag( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions: Vec, + encoded_transactions: Option>, + rbc: StarfishRbcFieldsV3, + ) -> Self { + Self::new_starfish_rbc_with_fields( + authority, + round, + block_references, + acknowledgment_references, + meta_creation_time_ns, + transactions, + encoded_transactions, + Some(rbc), + ) + } + + #[allow(clippy::too_many_arguments)] + fn new_starfish_rbc_with_fields( + authority: AuthorityIndex, + round: RoundNumber, + block_references: Vec, + acknowledgment_references: Vec, + meta_creation_time_ns: TimestampNs, + transactions: Vec, + encoded_transactions: Option>, + starfish_rbc_v3: Option, ) -> Self { let transactions_commitment = if let Some(ref encoded) = encoded_transactions { TransactionsCommitment::new_from_encoded_transactions(encoded, authority as usize).0 @@ -868,14 +1008,25 @@ impl VerifiedBlock { compress_acknowledgments(&block_references, &acknowledgment_references); let logical_acknowledgments = expand_acknowledgments(&block_references, intersection, &extra_references); - let digest = BlockDigest::new_starfish_rbc_header( - authority, - round, - &block_references, - &logical_acknowledgments, - meta_creation_time_ns, - transactions_commitment, - ); + let digest = match starfish_rbc_v3.as_ref() { + Some(rbc) => BlockDigest::new_starfish_rbc_single_dag_header( + authority, + round, + &block_references, + &logical_acknowledgments, + meta_creation_time_ns, + transactions_commitment, + rbc, + ), + None => BlockDigest::new_starfish_rbc_header( + authority, + round, + &block_references, + &logical_acknowledgments, + meta_creation_time_ns, + transactions_commitment, + ), + }; let transaction_data = (!transactions.is_empty()).then(|| TransactionData::new(transactions)); Self { @@ -897,6 +1048,7 @@ impl VerifiedBlock { bls: None, sailfish: None, unprovable_certificate: None, + starfish_rbc_v3, serialized: None, }, transaction_data, @@ -980,6 +1132,7 @@ impl VerifiedBlock { bls: bls.map(Box::new), sailfish: sailfish.map(Box::new), unprovable_certificate, + starfish_rbc_v3: None, serialized: None, }; @@ -1020,6 +1173,7 @@ impl VerifiedBlock { bls: None, sailfish: None, unprovable_certificate: None, + starfish_rbc_v3: None, serialized: None, }; let mut block = Self { @@ -1243,6 +1397,7 @@ impl VerifiedBlock { bls: bls.map(Box::new), sailfish: sailfish.map(Box::new), unprovable_certificate, + starfish_rbc_v3: None, serialized: None, }; @@ -1724,6 +1879,17 @@ impl VerifiedBlock { round ); } + ensure!( + consensus_protocol.is_starfish_rbc_single_dag() + == self.header.starfish_rbc_v3().is_some(), + "Only single-DAG Starfish-RBC blocks may carry V3 RBC reference evidence" + ); + if let Some(rbc) = self.header.starfish_rbc_v3() { + ensure!( + rbc.validate_for_block(committee, round), + "Single-DAG Starfish-RBC evidence is not canonical" + ); + } match consensus_protocol { ConsensusProtocol::StarfishBls => { ensure!( @@ -1916,6 +2082,7 @@ impl VerifiedBlock { } ConsensusProtocol::Starfish | ConsensusProtocol::StarfishRbc + | ConsensusProtocol::StarfishRbcSingleDag | ConsensusProtocol::StarfishSpeed => { ensure!( threshold_clock_valid_block_header(&self.header, committee), @@ -3185,6 +3352,55 @@ mod tests { ); } + #[test] + fn single_dag_rbc_uses_one_versioned_block_identity_and_canonical_evidence() { + let committee = Committee::new_for_benchmarks(4); + let parents: Vec<_> = committee + .authorities() + .map(|authority| BlockReference::new_test(authority, 1)) + .collect(); + let echo = StarfishRbcReferenceV3::new( + StarfishRbcReferenceKindV3::Echo, + BlockReference::new_test(3, 1), + ); + let ready = StarfishRbcReferenceV3::new( + StarfishRbcReferenceKindV3::Ready, + BlockReference::new_test(1, 1), + ); + let v3 = VerifiedBlock::new_starfish_rbc_single_dag( + 0, + 2, + parents.clone(), + Vec::new(), + 17, + Vec::new(), + None, + StarfishRbcFieldsV3::new(vec![ready, echo]), + ); + let v3_reordered = VerifiedBlock::new_starfish_rbc_single_dag( + 0, + 2, + parents.clone(), + Vec::new(), + 17, + Vec::new(), + None, + StarfishRbcFieldsV3::new(vec![echo, ready]), + ); + let direct = + VerifiedBlock::new_starfish_rbc(0, 2, parents, Vec::new(), 17, Vec::new(), None); + + assert_eq!(v3.reference(), v3_reordered.reference()); + assert_ne!(v3.reference(), direct.reference()); + let canonical = RbcCanonicalHeader::from_block_header(v3.header()).unwrap(); + canonical.validate_for_committee(&committee).unwrap(); + assert_eq!(canonical.reference(), *v3.reference()); + assert_eq!( + canonical.starfish_rbc_v3().unwrap().references(), + &[echo, ready] + ); + } + #[test] fn starfish_rbc_carrier_verification_is_content_only() { let committee = Committee::new_for_benchmarks(4); @@ -3257,6 +3473,7 @@ mod tests { bls: None, sailfish: None, unprovable_certificate: None, + starfish_rbc_v3: None, serialized: None, }; diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index d8c014ab..57873315 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -150,6 +150,28 @@ impl Validator { "Starfish-RBC-DAG vote-QC fast path is restricted to finite testbed benchmarks" )); } + if public_config + .parameters + .starfish_rbc_single_dag_echo_qc_fast_path + && !protocol_config + .consensus_protocol + .is_starfish_rbc_single_dag() + { + return Err(eyre!( + "Starfish-RBC single-DAG ECHO-QC fast path requires consensus \ + 'starfish-rbc-single-dag'" + )); + } + if public_config + .parameters + .starfish_rbc_single_dag_echo_qc_fast_path + && parameters.benchmark_duration.is_none() + { + return Err(eyre!( + "Starfish-RBC single-DAG ECHO-QC fast path is restricted to finite testbed \ + benchmarks" + )); + } if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { return Err(eyre!( "Starfish-RBC-DAG autonomous clock requires consensus 'starfish-rbc'" diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index bbd1091e..01847d74 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -210,6 +210,11 @@ enum Operation { /// Requires embedded RBC-DAG authority and changes the finality proof. #[clap(long, default_value_t = false)] starfish_rbc_dag_vote_qc_fast_path: bool, + /// Testbed-only: deliver a single-DAG RBC header after quorum ECHO. + /// This preserves uniqueness but not Byzantine selective-withholding + /// totality, so it is restricted to finite benchmark runs. + #[clap(long, default_value_t = false)] + starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. /// The physical carrier heartbeat remains on `leader_timeout`. #[clap(long, value_name = "INT")] @@ -337,6 +342,7 @@ async fn main() -> Result<()> { starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_dag_vote_qc_fast_path, + starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, duration_secs, @@ -355,11 +361,13 @@ async fn main() -> Result<()> { node_parameters.starfish_rbc_dag_embedded_rbc_authority = starfish_rbc_dag_embedded_rbc_authority; node_parameters.starfish_rbc_dag_vote_qc_fast_path = starfish_rbc_dag_vote_qc_fast_path; + node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = + starfish_rbc_single_dag_echo_qc_fast_path; node_parameters.starfish_rbc_dag_consensus_timeout = starfish_rbc_dag_consensus_timeout_ms.map(Duration::from_millis); node_parameters.starfish_rbc_dag_shadow_buffered_wal = starfish_rbc_dag_shadow_buffered_wal; - if consensus_protocol == "starfish-rbc" { + if is_starfish_rbc_selection(&consensus_protocol) { node_parameters.refresh_starfish_rbc_protocol_instance(); } if let Some(ref mode) = dissemination_mode { @@ -606,6 +614,11 @@ async fn local_benchmark( } ); } + if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { + println!( + "Single-DAG ECHO-QC delivery: ENABLED (testbed-only; Byzantine totality not claimed)" + ); + } if let Some(latency) = node_parameters.uniform_latency_ms { println!("Network Latency: {latency} ms (uniform)"); } else { @@ -1330,13 +1343,20 @@ fn ensure_starfish_rbc_protocol_instance( consensus_protocol: &str, node_parameters: &mut NodeParameters, ) { - if consensus_protocol == "starfish-rbc" + if is_starfish_rbc_selection(consensus_protocol) && node_parameters.starfish_rbc_protocol_instance.is_none() { node_parameters.refresh_starfish_rbc_protocol_instance(); } } +fn is_starfish_rbc_selection(consensus_protocol: &str) -> bool { + matches!( + consensus_protocol, + "starfish-rbc" | "starfish-rbc-single-dag" + ) +} + fn validate_local_benchmark_port_offset( public_config: &NodePublicConfig, port_offset: u16, @@ -1538,6 +1558,7 @@ mod tests { "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-dag-vote-qc-fast-path", + "--starfish-rbc-single-dag-echo-qc-fast-path", "--starfish-rbc-dag-consensus-timeout-ms", "250", "--starfish-rbc-dag-shadow-buffered-wal", @@ -1553,6 +1574,7 @@ mod tests { starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_dag_vote_qc_fast_path, + starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, port_offset, @@ -1567,6 +1589,7 @@ mod tests { assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_dag_embedded_rbc_authority); assert!(starfish_rbc_dag_vote_qc_fast_path); + assert!(starfish_rbc_single_dag_echo_qc_fast_path); assert_eq!(starfish_rbc_dag_consensus_timeout_ms, Some(250)); assert!(starfish_rbc_dag_shadow_buffered_wal); assert_eq!(port_offset, 2500); diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index b5808f5a..1fbe5b3c 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -1,5 +1,11 @@ # Starfish-RBC-DAG protocol design +> **Frozen comparison baseline.** This document specifies the experimental +> two-plane carrier implementation. New development targets the one-block, +> one-identity design in `starfish-rbc-single-dag-v3.md`; V1/V2 carrier and +> projection formats remain available only for compatibility and matched +> performance comparisons. + Status: standalone MAC-vector RBC-DAG prototype with authoritative optimistic delivery and committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoint transfer, and full validator recovery remain incomplete diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md new file mode 100644 index 00000000..1e6f2e78 --- /dev/null +++ b/docs/starfish-rbc-single-dag-v3.md @@ -0,0 +1,127 @@ +# Single-DAG Starfish-RBC V3 + +## Status + +This is the active experimental successor to the two-plane carrier prototype +documented in `starfish-rbc-dag-protocol.md`. It is selected as +`starfish-rbc-single-dag`. The old implementation remains available as a +benchmark baseline; its carrier and projection formats are frozen and are not +reinterpreted as V3. + +V3 is a research-testbed protocol. Crash recovery, bounded retirement and the +complete asynchronous safety/liveness proof remain required before production +use. + +## One identity and one DAG + +Every protocol vertex is an ordinary Starfish `VerifiedBlock`. Its canonical +`BlockReference` simultaneously identifies: + +1. the author's reliable-broadcast proposal; +2. the dirty/authenticated DAG vertex; +3. the delivered/clean DAG vertex; and +4. the vertex considered by the existing Starfish committer. + +V3 has no `CandidateCarrierV1`, nested `ConsensusVertexV1`, carrier clock, +delivery frontier, or physical-to-logical projection. Dirty and clean are +monotone states of the same vertex, not separate DAG identities. + +## Typed RBC references + +The V3 block header contains a canonical `StarfishRbcFieldsV3` list. Each item +is one of: + +- `Echo(BlockReference)`; or +- `Ready(BlockReference)`. + +The carrying block's authenticated author is the statement sender. The list is +part of that ordinary block's content digest. Consequently no standalone phase +MAC, signature, or phase network message is needed. + +References are sorted, duplicate-free and bounded by `6 * committee_size` per +block. A sender may name at most one digest for each `(phase, target author, +target round)`. Targets must be non-genesis, known-authority references no newer +than the carrying block. + +## Progress and cleanliness + +Block production follows the authenticated dirty threshold clock: a V3 block +may causally reference authenticated previous-round blocks that have not yet +been RBC-delivered. This is necessary to prevent a circular wait in which RBC +evidence needs a later block while later blocks require earlier RBC delivery. + +The existing clean DAG remains fail-closed. A block becomes consensus-visible +only after: + +1. its own RBC instance delivers the exact canonical header; +2. its payload is data-available where required; and +3. every causal parent and ordering acknowledgment is clean. + +Typed RBC evidence references are testimony, not causal parents. Missing or +selectively supplied Byzantine evidence therefore cannot contaminate the +carrying block's clean dependency cone. + +## Communication + +Normal communication consists only of ordinary Starfish block proposals. RBC +ECHO/READY statements ride in later blocks. Header, payload and missing-parent +requests are synchronization/recovery traffic and remain permitted. A V3 node +must neither emit nor count the legacy standalone `RbcPhase` messages. + +## Initial pipeline + +After authenticating a proposal, a validator locks an ECHO and queues its typed +reference for the next ordinary block. Observing quorum ECHO references locks a +READY reference for a later ordinary block. Quorum READY references deliver the +original block. These waves pipeline across normal Starfish rounds; they do not +create a second physical round counter. + +## Safety boundaries + +- Block authentication and the V3 digest bind every embedded statement to its + sender. +- Sender locks prohibit conflicting ECHO or READY references for one target + slot. +- RBC delivery never follows from dirty-DAG admission alone. +- Consensus parent selection and commitment use only clean vertices. +- Recovery content must recompute to the exact requested `BlockReference`. +- Frozen direct-RBC and carrier-DAG formats retain their old domains. + +### Explicit testbed ECHO-QC fast path + +Finite benchmarks may opt into +`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode a node delivers an +exact header as soon as it has quorum locked ECHO statements, while still +emitting the normal READY statement. Quorum intersection prevents two +different headers from both obtaining such a certificate, so agreement is +preserved. The current wire format does not carry the exact ECHO witness set, +however: Byzantine ECHO senders can selectively reveal their statements so +one honest node obtains quorum while another never can. Consequently the flag +does **not** provide the normal RBC totality guarantee and is rejected for +unbounded/production runs. The default V3 path continues to require quorum +READY. + +## Required validation + +Before using V3 benchmark results, tests must cover canonical identity, +conflicting sender locks, dirty-clock progress, clean dependency closure, +embedded ECHO/READY delivery, absence of normal phase messages, Byzantine +withholding with bounded recovery, restart replay and deterministic Starfish +commit order. Matched n=10 and n=40 zero/AWS runs must compare V3 against the +frozen carrier baseline using identical load and duration. + +## Initial n=10 testbed checkpoint + +The first matched local runs used 1,000 offered transactions/s for 20 seconds +with MAC authentication and the fixed 50 ms V3 round limiter: + +| Protocol/profile | Network | p50 E2E | Eventual TPS | Outbound/node | +| --- | ---: | ---: | ---: | ---: | +| `starfish-rbc-single-dag` | zero | 498.3 ms | 1,000 | 0.79 MB/s | +| `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | +| V3 + flagged ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | +| `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | + +All exact offered transactions committed during the bounded drain. These are +single-machine research measurements, not production claims. In particular, +the 961 ms row carries the ECHO-QC totality limitation above. From 2b423aaf21b5b3a7c47b046a3718fb8249f1ead0 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:48:31 +0200 Subject: [PATCH 45/62] record 40-validator single-DAG result --- docs/starfish-rbc-single-dag-v3.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index 1e6f2e78..de2e058d 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -120,8 +120,9 @@ with MAC authentication and the fixed 50 ms V3 round limiter: | `starfish-rbc-single-dag` | zero | 498.3 ms | 1,000 | 0.79 MB/s | | `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | | V3 + flagged ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | +| V3 + flagged ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | | `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | All exact offered transactions committed during the bounded drain. These are single-machine research measurements, not production claims. In particular, -the 961 ms row carries the ECHO-QC totality limitation above. +both ECHO-QC rows carry the totality limitation above. From 37379aa46cbb849fe5446be7a56ae9bc53dc4469 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:43:58 +0200 Subject: [PATCH 46/62] add portable single-DAG ECHO QC --- crates/starfish-core/src/config.rs | 7 +- crates/starfish-core/src/core.rs | 51 +- .../starfish-core/src/core_thread/spawned.rs | 35 ++ crates/starfish-core/src/crypto.rs | 41 ++ crates/starfish-core/src/net_sync.rs | 14 +- crates/starfish-core/src/starfish_rbc.rs | 457 ++++++++++++++++-- .../starfish-core/src/starfish_rbc_service.rs | 130 ++++- crates/starfish-core/src/syncer.rs | 10 + crates/starfish-core/src/types.rs | 153 +++++- crates/starfish/src/main.rs | 7 +- docs/starfish-rbc-single-dag-v3.md | 49 +- 11 files changed, 880 insertions(+), 74 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index fad2d43e..7701db4d 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -94,10 +94,9 @@ pub struct NodeParameters { /// Starfish finality proof shape. #[serde(default)] pub starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only single-DAG RBC path: deliver an exact header after quorum - /// ECHO rather than quorum READY. Quorum intersection preserves a unique - /// value, but selective Byzantine ECHO withholding can violate totality; - /// this must remain an explicit benchmark flag. + /// Testbed single-DAG RBC path: publicly signed ECHO votes are copied into + /// a portable quorum certificate carried by an ordinary DAG block. This + /// preserves uniqueness and totality without a standalone phase message. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index d2970e8b..fe8b6fc9 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -39,7 +39,8 @@ use crate::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishFields, Shard, - StarfishRbcFieldsV3, StarfishRbcReferenceV3, VerifiedBlock, + StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, StarfishRbcReferenceV3, + VerifiedBlock, }, }; @@ -57,6 +58,8 @@ pub struct Core { /// Irrevocable local ECHO/READY statements waiting to ride on the next /// ordinary block in the single-DAG protocol. pending_starfish_rbc_references: BTreeSet, + pending_starfish_rbc_echo_votes: BTreeSet, + pending_starfish_rbc_echo_qcs: BTreeSet, // For Byzantine node, last_own_block contains a vector of blocks last_own_block: Vec, block_handler: H, @@ -323,6 +326,8 @@ impl Core { pending, pending_reconstructed_data: AHashMap::new(), pending_starfish_rbc_references: BTreeSet::new(), + pending_starfish_rbc_echo_votes: BTreeSet::new(), + pending_starfish_rbc_echo_qcs: BTreeSet::new(), last_own_block: vec![last_own_block], block_handler, authority, @@ -358,6 +363,10 @@ impl Core { &self.signer } + pub fn get_bls_signer(&self) -> &BlsSigner { + &self.bls_signer + } + pub(crate) fn add_starfish_rbc_reference(&mut self, reference: StarfishRbcReferenceV3) { assert!( self.dag_state @@ -368,6 +377,24 @@ impl Core { self.pending_starfish_rbc_references.insert(reference); } + pub(crate) fn add_starfish_rbc_echo_vote(&mut self, vote: StarfishRbcEchoVoteV3) { + assert!( + self.dag_state + .consensus_protocol + .is_starfish_rbc_single_dag() + ); + self.pending_starfish_rbc_echo_votes.insert(vote); + } + + pub(crate) fn add_starfish_rbc_echo_qc(&mut self, qc: StarfishRbcEchoQcV3) { + assert!( + self.dag_state + .consensus_protocol + .is_starfish_rbc_single_dag() + ); + self.pending_starfish_rbc_echo_qcs.insert(qc); + } + pub(crate) fn get_ml_dsa_44_signer(&self) -> &crate::crypto::MlDsa44Signer { &self.ml_dsa_44_signer } @@ -857,7 +884,27 @@ impl Core { for reference in &references { self.pending_starfish_rbc_references.remove(reference); } - StarfishRbcFieldsV3::new(references) + let echo_votes: Vec<_> = self + .pending_starfish_rbc_echo_votes + .iter() + .filter(|vote| vote.target().round <= clock_round) + .take(self.committee.len().saturating_mul(3)) + .copied() + .collect(); + for vote in &echo_votes { + self.pending_starfish_rbc_echo_votes.remove(vote); + } + let echo_qcs: Vec<_> = self + .pending_starfish_rbc_echo_qcs + .iter() + .filter(|qc| qc.target().round < clock_round) + .take(self.committee.len()) + .cloned() + .collect(); + for qc in &echo_qcs { + self.pending_starfish_rbc_echo_qcs.remove(qc); + } + StarfishRbcFieldsV3::with_portable_echo(references, echo_votes, echo_qcs) }); // Create and store blocks diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 65405824..86728659 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -82,6 +82,8 @@ enum CoreThreadCommand { /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), + ApplyStarfishRbcEchoVote(crate::types::StarfishRbcEchoVoteV3, oneshot::Sender<()>), + ApplyStarfishRbcEchoQc(crate::types::StarfishRbcEchoQcV3, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. ApplyStarfishRbcDagFrontier( CommittedFrontierDeltaV1, @@ -285,6 +287,23 @@ impl CoreThread { self.syncer.apply_starfish_rbc_reference(reference); sender.send(()).ok(); } + CoreThreadCommand::ApplyStarfishRbcEchoVote(vote, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_echo_vote"]) + .inc(); + self.syncer.apply_starfish_rbc_echo_vote(vote); + sender.send(()).ok(); + } + CoreThreadCommand::ApplyStarfishRbcEchoQc(qc, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_echo_qc"]) + .inc(); + self.syncer.apply_starfish_rbc_echo_qc(qc); + sender.send(()).ok(); + } CoreThreadCommand::ApplyStarfishRbcDagFrontier(delta, sender) => { metrics .core_thread_tasks_total diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 3d85e0bf..27e38796 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -323,6 +323,30 @@ impl BlockDigest { hasher.update(&[evidence.kind().tag()]); hash_reference(&mut hasher, &evidence.reference()); } + // Preserve the frozen V3 digest exactly when the portable extension + // is absent. A non-empty extension is explicitly tagged before its + // length-delimited fields, so old stored/default blocks still reopen. + if !rbc.echo_votes().is_empty() || !rbc.echo_qcs().is_empty() { + hasher.update(b"PORTABLE_ECHO_QC_V1"); + let vote_len = u32::try_from(rbc.echo_votes().len()) + .expect("Starfish-RBC ECHO vote count exceeds u32"); + hasher.update(&vote_len.to_be_bytes()); + for vote in rbc.echo_votes() { + hash_reference(&mut hasher, &vote.target()); + hasher.update(&vote.sender().to_be_bytes()); + hasher.update(vote.signature().as_ref()); + } + let qc_len = u32::try_from(rbc.echo_qcs().len()) + .expect("Starfish-RBC ECHO-QC count exceeds u32"); + hasher.update(&qc_len.to_be_bytes()); + for qc in rbc.echo_qcs() { + hash_reference(&mut hasher, &qc.target()); + for word in qc.signers().words() { + hasher.update(&word.to_be_bytes()); + } + hasher.update(qc.signature().as_ref()); + } + } Self(hasher.finalize().into()) } @@ -1363,6 +1387,23 @@ pub fn bls_aggregate(sigs: &[&BlsSignatureBytes]) -> BlsSignatureBytes { BlsSignatureBytes(agg.to_signature().to_bytes()) } +/// Fallible counterpart used on untrusted wire votes before they have been +/// individually verified. The returned aggregate is still untrusted and must +/// be verified against the exact signer set and message. +pub fn bls_try_aggregate(sigs: &[&BlsSignatureBytes]) -> Option { + if sigs.is_empty() { + return None; + } + let parsed: Vec = sigs + .iter() + .map(|signature| bls::Signature::from_bytes(&signature.0)) + .collect::>() + .ok()?; + let references: Vec<_> = parsed.iter().collect(); + let aggregate = bls::AggregateSignature::aggregate(&references, true).ok()?; + Some(BlsSignatureBytes(aggregate.to_signature().to_bytes())) +} + /// Verify an aggregate signature against multiple public keys (all signed same /// message). #[allow(dead_code)] diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 9f4ec5e7..1c95a86f 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -3192,7 +3192,10 @@ impl NetworkSyncer BlockAuthenticationScheme::MlDsa65 => { RbcInitialAuthenticator::MlDsa65(core.get_ml_dsa_65_signer().clone()) } - BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac( + core.get_signer().clone(), + core.get_bls_signer().clone(), + ), }; let (service, events, task) = start_starfish_rbc_service_with_phase_authority( committee.clone(), @@ -3568,6 +3571,15 @@ impl NetworkSyncer .apply_starfish_rbc_reference(reference) .await; } + RbcServiceEvent::EchoVoteReady(vote) => { + event_inner + .syncer + .apply_starfish_rbc_echo_vote(vote) + .await; + } + RbcServiceEvent::EchoQcReady(qc) => { + event_inner.syncer.apply_starfish_rbc_echo_qc(qc).await; + } RbcServiceEvent::Rejected { peer, error } => { tracing::warn!( "Rejected Starfish-RBC input from {:?}: {}", diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index 4096a982..feeb9446 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -16,13 +16,15 @@ use crate::{ committee::{Committee, QuorumThreshold, StakeAggregator, ValidityThreshold}, crypto::{ Blake3Hasher, MacKey, MacTag, MlDsa44SignatureBytes, MlDsa65SignatureBytes, SignatureBytes, - TransactionsCommitment, + TransactionsCommitment, bls_aggregate, bls_fast_aggregate_verify, + bls_public_keys_for_signers, bls_try_aggregate, }, types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, - TransactionData, VerifiedBlock, compress_acknowledgments, expand_acknowledgments, + StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, + StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, + VerifiedBlock, compress_acknowledgments, expand_acknowledgments, }, }; @@ -31,6 +33,7 @@ const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_V1_COMMITTEE_ID"; const INITIAL_KIND: u8 = 0x00; const ECHO_KIND: u8 = 0x01; const READY_KIND: u8 = 0x02; +const PORTABLE_ECHO_KIND: u8 = 0x03; const PROTOCOL_INSTANCE_SIZE: usize = 32; const COMMITTEE_ID_SIZE: usize = 32; @@ -523,9 +526,27 @@ impl RbcCanonicalHeader { ) }) .ok_or(RbcError::HeaderContentTooLarge)?; + let portable_echo_bytes = self.starfish_rbc_v3.as_ref().map_or(0, |rbc| { + let vote_bytes: usize = rbc + .echo_votes() + .iter() + .map(|vote| RBC_BLOCK_REFERENCE_SIZE + 2 + vote.signature().as_ref().len()) + .sum(); + let qc_bytes: usize = rbc + .echo_qcs() + .iter() + .map(|qc| { + RBC_BLOCK_REFERENCE_SIZE + + qc.signers().words().len() * std::mem::size_of::() + + qc.signature().as_ref().len() + }) + .sum(); + vote_bytes.saturating_add(qc_bytes) + }); RBC_BLOCK_REFERENCE_SIZE .checked_mul(reference_count) .and_then(|size| size.checked_add(RBC_HEADER_FIXED_CONTENT_SIZE)) + .and_then(|size| size.checked_add(portable_echo_bytes)) .ok_or(RbcError::HeaderContentTooLarge) } } @@ -892,6 +913,8 @@ pub(crate) enum RbcEffect { holders: AuthoritySet, }, Deliver(PinnedRbcHeader), + PortableEchoVote(BlockReference), + PortableEchoQc(StarfishRbcEchoQcV3), } #[derive(Clone, Debug, Eq, PartialEq)] @@ -958,6 +981,8 @@ pub(crate) enum RbcError { DuplicateAcknowledgment(BlockReference), InvalidThresholdClock, InvalidSingleDagEvidence, + InvalidPortableEchoVote, + InvalidPortableEchoQc, HeaderDigestMismatch { expected: BlockDigest, actual: BlockDigest, @@ -1099,6 +1124,12 @@ impl fmt::Display for RbcError { Self::InvalidSingleDagEvidence => { f.write_str("Starfish-RBC V3 block carries non-canonical reference evidence") } + Self::InvalidPortableEchoVote => { + f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO vote") + } + Self::InvalidPortableEchoQc => { + f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO-QC") + } Self::HeaderDigestMismatch { expected, actual } => write!( f, "Starfish-RBC header digest mismatch: expected {expected}, got {actual}" @@ -1181,6 +1212,10 @@ struct CandidateState { ready_validity_observed: bool, ready_quorum_observed: bool, header_request_holders: AuthoritySet, + portable_echo_votes: BTreeMap, + portable_echo_votes_verified: bool, + portable_echo_qc_emitted: bool, + portable_echo_qc_observed: bool, } impl CandidateState { @@ -1193,6 +1228,10 @@ impl CandidateState { ready_validity_observed: false, ready_quorum_observed: false, header_request_holders: AuthoritySet::default(), + portable_echo_votes: BTreeMap::new(), + portable_echo_votes_verified: false, + portable_echo_qc_emitted: false, + portable_echo_qc_observed: false, } } @@ -1256,6 +1295,7 @@ enum ProgressAction { NeedHeader(AuthoritySet), SendReady, Deliver, + EmitPortableEchoQc(Vec), None, } @@ -1266,10 +1306,8 @@ pub(crate) struct StarfishRbcKernel { mac_keys: Arc>, local_round: RoundNumber, minimum_new_slot_round: RoundNumber, - /// Testbed-only optimistic path. A quorum of locked ECHOs proves a unique - /// value, but without a portable proof it does not prove that every honest - /// node can assemble the same quorum under selective Byzantine - /// withholding. Keep disabled for the asynchronous RBC contract. + /// Testbed portable fast path. Public ECHO signatures are copied into a + /// quorum certificate carried by an ordinary single-DAG block. echo_qc_fast_path: bool, slots: BTreeMap>, } @@ -1641,9 +1679,13 @@ impl StarfishRbcKernel { .echoes .add(own_authority, &committee); - let mut effects = vec![RbcEffect::MulticastPhase { - phase: RbcPhase::Echo, - block_ref, + let mut effects = vec![if self.echo_qc_fast_path { + RbcEffect::PortableEchoVote(block_ref) + } else { + RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, + } }]; effects.extend(self.drive(block_ref)); Ok(effects) @@ -1712,6 +1754,175 @@ impl StarfishRbcKernel { Ok(self.drive(block_ref)) } + /// Digest signed by an ECHO sender for the portable single-DAG fast path. + /// The protocol instance and committee identifier prevent cross-run reuse. + pub(crate) fn portable_echo_signature_digest( + &self, + target: BlockReference, + ) -> Result<[u8; 32], RbcError> { + self.validate_block_ref(&target)?; + Ok(blake3::hash(&encode_base_statement( + &self.context, + PORTABLE_ECHO_KIND, + &target, + )) + .into()) + } + + pub(crate) fn handle_portable_echo_vote( + &mut self, + vote: StarfishRbcEchoVoteV3, + ) -> Result, RbcError> { + if !self.echo_qc_fast_path { + return Err(RbcError::InvalidPortableEchoVote); + } + self.validate_block_ref(&vote.target())?; + if self + .candidate(&vote.target()) + .is_some_and(|candidate| candidate.portable_echo_votes_verified) + { + return Ok(Vec::new()); + } + let candidate = self.candidate_mut(vote.target()); + match candidate.portable_echo_votes.get(&vote.sender()) { + Some(existing) if existing != &vote => return Err(RbcError::InvalidPortableEchoVote), + Some(_) => return Ok(Vec::new()), + None => { + candidate.portable_echo_votes.insert(vote.sender(), vote); + } + } + self.verify_portable_echo_vote_batch(vote.target())?; + Ok(self.drive(vote.target())) + } + + fn verify_portable_echo_vote_batch(&mut self, target: BlockReference) -> Result<(), RbcError> { + let votes: Vec<_> = self + .candidate(&target) + .into_iter() + .flat_map(|candidate| candidate.portable_echo_votes.values().copied()) + .collect(); + let stake: Stake = votes + .iter() + .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) + .sum(); + if !self.committee.is_quorum(stake) { + return Ok(()); + } + let digest = self.portable_echo_signature_digest(target)?; + let batch_valid = |votes: &[StarfishRbcEchoVoteV3]| { + let mut signers = AuthoritySet::default(); + for vote in votes { + signers.insert(vote.sender()); + } + let signatures: Vec<_> = votes.iter().map(|vote| vote.signature()).collect(); + let signature_refs: Vec<_> = signatures.iter().collect(); + let Some(aggregate) = bls_try_aggregate(&signature_refs) else { + return false; + }; + let Some(public_keys) = bls_public_keys_for_signers(&self.committee, signers) else { + return false; + }; + bls_fast_aggregate_verify(&digest, &aggregate, &public_keys) + }; + let valid_votes = if batch_valid(&votes) { + votes + } else { + votes + .into_iter() + .filter(|vote| { + self.committee + .get_bls_public_key(vote.sender()) + .is_some_and(|public_key| { + public_key + .verify_trusted(&digest, &vote.signature()) + .is_ok() + }) + }) + .collect() + }; + let valid_stake: Stake = valid_votes + .iter() + .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) + .sum(); + let committee = Arc::clone(&self.committee); + let slot = self.slot_mut(target); + if committee.is_quorum(valid_stake) { + for vote in &valid_votes { + slot.record_phase_sender(RbcPhase::Echo, vote.sender(), target); + } + } + let candidate = slot + .candidates + .entry(target) + .or_insert_with(CandidateState::new); + candidate.portable_echo_votes = valid_votes + .iter() + .map(|vote| (vote.sender(), *vote)) + .collect(); + if committee.is_quorum(valid_stake) { + candidate.portable_echo_votes_verified = true; + for vote in valid_votes { + candidate.echoes.add(vote.sender(), &committee); + } + } + Ok(()) + } + + pub(crate) fn handle_portable_echo_qc( + &mut self, + qc: &StarfishRbcEchoQcV3, + ) -> Result, RbcError> { + if !self.echo_qc_fast_path { + return Err(RbcError::InvalidPortableEchoQc); + } + self.validate_block_ref(&qc.target())?; + if self + .candidate(&qc.target()) + .is_some_and(|candidate| candidate.portable_echo_qc_observed) + { + return Ok(self.drive(qc.target())); + } + let mut stake = 0; + for sender in qc.signers().present() { + stake += self + .committee + .get_stake(sender) + .ok_or(RbcError::InvalidPortableEchoQc)?; + } + if !self.committee.is_quorum(stake) { + return Err(RbcError::InvalidPortableEchoQc); + } + let public_keys = bls_public_keys_for_signers(&self.committee, qc.signers()) + .ok_or(RbcError::InvalidPortableEchoQc)?; + if !bls_fast_aggregate_verify( + &self.portable_echo_signature_digest(qc.target())?, + &qc.signature(), + &public_keys, + ) { + return Err(RbcError::InvalidPortableEchoQc); + } + let candidate = self.candidate_mut(qc.target()); + candidate.portable_echo_qc_observed = true; + let relay = if candidate.portable_echo_qc_emitted { + None + } else { + candidate.portable_echo_qc_emitted = true; + Some(RbcEffect::PortableEchoQc(qc.clone())) + }; + let mut effects = relay.into_iter().collect::>(); + effects.extend(self.drive(qc.target())); + if self + .candidate(&qc.target()) + .is_some_and(|candidate| candidate.header.is_none()) + { + effects.push(RbcEffect::NeedHeader { + block_ref: qc.target(), + holders: qc.signers(), + }); + } + Ok(effects) + } + /// Materialize one recipient-specific message for an untagged multicast /// effect. The network adapter calls this once per non-local recipient. pub(crate) fn make_phase_message( @@ -2030,6 +2241,7 @@ impl StarfishRbcKernel { } fn drive(&mut self, block_ref: BlockReference) -> Vec { + let committee = Arc::clone(&self.committee); let validity_threshold = self.committee.validity_threshold(); let quorum_threshold = self.committee.quorum_threshold(); let echo_qc_fast_path = self.echo_qc_fast_path; @@ -2048,19 +2260,37 @@ impl StarfishRbcKernel { let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; + let portable_vote_stake = candidate + .portable_echo_votes + .keys() + .map(|sender| committee.get_stake(*sender).unwrap_or_default()) + .sum::(); + let portable_qc_ready = echo_qc_fast_path + && !candidate.portable_echo_qc_emitted + && candidate.portable_echo_votes_verified + && committee.is_quorum(portable_vote_stake); let blocked_on_header = candidate.header.is_none() && ((can_send_ready && ready_trigger) - || (can_deliver && candidate.ready_quorum_observed)); + || (can_deliver + && (candidate.ready_quorum_observed + || candidate.portable_echo_qc_observed))); let holders = candidate.holders(); - if blocked_on_header && holders != candidate.header_request_holders { + if portable_qc_ready { + ProgressAction::EmitPortableEchoQc( + candidate.portable_echo_votes.values().copied().collect(), + ) + } else if blocked_on_header && holders != candidate.header_request_holders { candidate.header_request_holders = holders; ProgressAction::NeedHeader(holders) - } else if candidate.header.is_some() && can_send_ready && ready_trigger { + } else if candidate.header.is_some() + && can_send_ready + && ready_trigger + && !(echo_qc_fast_path && candidate.portable_echo_qc_emitted) + { ProgressAction::SendReady } else if candidate.header.is_some() && can_deliver - && (candidate.ready_quorum_observed - || (echo_qc_fast_path && candidate.echo_quorum_observed)) + && (candidate.ready_quorum_observed || candidate.portable_echo_qc_observed) { ProgressAction::Deliver } else { @@ -2105,6 +2335,29 @@ impl StarfishRbcKernel { effects.push(RbcEffect::Deliver(header)); } } + ProgressAction::EmitPortableEchoQc(votes) => { + let candidate = self.candidate_mut(block_ref); + if !candidate.portable_echo_qc_emitted { + candidate.portable_echo_qc_emitted = true; + // The service emits this QC before the following + // delivery effect. Its ordered Core bridge awaits + // creation/dissemination of the QC-bearing block, so + // delivery cannot overtake portable publication. + candidate.portable_echo_qc_observed = true; + let mut signers = AuthoritySet::default(); + for vote in &votes { + signers.insert(vote.sender()); + } + let signatures: Vec<_> = + votes.iter().map(|vote| vote.signature()).collect(); + let signature_refs: Vec<_> = signatures.iter().collect(); + effects.push(RbcEffect::PortableEchoQc(StarfishRbcEchoQcV3::new( + block_ref, + signers, + bls_aggregate(&signature_refs), + ))); + } + } ProgressAction::None => break, } } @@ -2214,7 +2467,8 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, + mac_keyrings_for_test, }, types::{BlockDigest, BlockReference}, }; @@ -2265,7 +2519,7 @@ mod tests { #[test] fn embedded_block_references_drive_rbc_without_phase_messages() { - let committee = Committee::new_for_benchmarks(4); + let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(committee.len()); let mut receiver = StarfishRbcKernel::new( committee.clone(), @@ -2318,8 +2572,8 @@ mod tests { } #[test] - fn flagged_echo_qc_fast_path_delivers_unique_header_without_ready_quorum() { - let committee = Committee::new_for_benchmarks(4); + fn portable_echo_qc_fast_path_requires_and_accepts_exact_signed_quorum() { + let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(committee.len()); let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( committee, @@ -2337,26 +2591,136 @@ mod tests { .unwrap(); receiver.authorize_echo(target).unwrap(); - let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); - assert!( - receiver - .handle_embedded_reference(1, echo) - .unwrap() - .is_empty() - ); - let effects = receiver.handle_embedded_reference(2, echo).unwrap(); + let digest = receiver.portable_echo_signature_digest(target).unwrap(); + let signer = dummy_bls_signer(); + let votes: Vec<_> = (0..3) + .map(|sender| StarfishRbcEchoVoteV3::new(target, sender, signer.sign_digest(&digest))) + .collect(); + for vote in &votes[..2] { + assert!( + !receiver + .handle_portable_echo_vote(*vote) + .unwrap() + .iter() + .any(|effect| matches!(effect, RbcEffect::Deliver(_))) + ); + } + let effects = receiver.handle_portable_echo_vote(votes[2]).unwrap(); - assert!(effects.iter().any(|effect| matches!( + assert!(!effects.iter().any(|effect| matches!( effect, RbcEffect::MulticastPhase { phase: RbcPhase::Ready, - block_ref, - } if *block_ref == target + .. + } + ))); + let qc = effects + .iter() + .find_map(|effect| match effect { + RbcEffect::PortableEchoQc(qc) => Some(qc.clone()), + _ => None, + }) + .expect("signed quorum must emit a portable ECHO-QC"); + let qc_position = effects + .iter() + .position(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) + .unwrap(); + let delivery_position = effects + .iter() + .position(|effect| { + matches!( + effect, + RbcEffect::Deliver(header) if header.reference() == target + ) + }) + .expect("delivery follows portable publication"); + assert!(qc_position < delivery_position); + assert!(receiver.handle_portable_echo_qc(&qc).unwrap().is_empty()); + + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(committee.len()); + let mut late_receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( + committee, + 3, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[3].clone()), + 1, + true, + ) + .unwrap(); + let missing = late_receiver.handle_portable_echo_qc(&qc).unwrap(); + assert!(missing.iter().any(|effect| matches!( + effect, + RbcEffect::PortableEchoQc(relayed) if relayed == &qc ))); + assert!(missing.iter().any(|effect| matches!( + effect, + RbcEffect::NeedHeader { block_ref, .. } if *block_ref == target + ))); + let effects = late_receiver + .note_header_available(pinned_header_for_context(late_receiver.context, target)) + .unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::Deliver(header) if header.reference() == target ))); + + let other_target = block(3, 1, 0x73); + let forged = StarfishRbcEchoQcV3::new(other_target, qc.signers(), qc.signature()); + assert_eq!( + receiver.handle_portable_echo_qc(&forged), + Err(RbcError::InvalidPortableEchoQc) + ); + } + + #[test] + fn invalid_echo_vote_cannot_poison_a_later_portable_quorum() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(committee.len()); + let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( + committee, + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + 1, + true, + ) + .unwrap(); + let target = block(3, 1, 0x74); + receiver + .note_header_available(pinned_header_for_context(receiver.context, target)) + .unwrap(); + let digest = receiver.portable_echo_signature_digest(target).unwrap(); + let signer = dummy_bls_signer(); + let wrong_signature = signer.sign_digest(&[0xFF; 32]); + for (sender, signature) in [ + (0, signer.sign_digest(&digest)), + (1, wrong_signature), + (2, signer.sign_digest(&digest)), + ] { + let effects = receiver + .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new(target, sender, signature)) + .unwrap(); + assert!( + !effects + .iter() + .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) + ); + } + let effects = receiver + .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new( + target, + 3, + signer.sign_digest(&digest), + )) + .unwrap(); + assert!( + effects + .iter() + .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) + ); } fn valid_canonical_header( @@ -2547,6 +2911,9 @@ mod tests { ); deliveries[owner as usize].push(header.reference()); } + RbcEffect::PortableEchoVote(_) | RbcEffect::PortableEchoQc(_) => { + panic!("portable ECHO effects require the single-DAG test harness") + } } } (deliveries, recoveries) @@ -2694,6 +3061,36 @@ mod tests { } } + #[test] + fn single_dag_digest_binds_portable_echo_votes_and_certificate() { + let target = block(0, 4, 0x43); + let signature = dummy_bls_signer().sign_digest(&[0x44; 32]); + let vote = StarfishRbcEchoVoteV3::new(target, 1, signature); + let mut signers = AuthoritySet::default(); + signers.insert(0); + signers.insert(1); + signers.insert(2); + let signatures = [&signature, &signature, &signature]; + let qc = StarfishRbcEchoQcV3::new(target, signers, bls_aggregate(&signatures)); + let empty = StarfishRbcFieldsV3::default(); + let with_vote = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), vec![vote], Vec::new()); + let with_qc = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), Vec::new(), vec![qc]); + let digest = |fields: &StarfishRbcFieldsV3| { + BlockDigest::new_starfish_rbc_single_dag_header( + 3, + 5, + &[], + &[], + 7, + TransactionsCommitment::default(), + fields, + ) + }; + assert_ne!(digest(&empty), digest(&with_vote)); + assert_ne!(digest(&empty), digest(&with_qc)); + assert_ne!(digest(&with_vote), digest(&with_qc)); + } + #[test] fn canonical_header_validation_is_authentication_independent_and_pins_content() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 8b08038a..90941860 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -25,7 +25,7 @@ use tokio::{ use crate::{ committee::Committee, - crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + crypto::{BlsSigner, MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, network::NetworkMessage, starfish_rbc::{ PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, @@ -34,8 +34,8 @@ use crate::{ }, types::{ AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - RoundNumber, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, - TimestampNs, TransactionData, + RoundNumber, StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, + StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, }, }; @@ -48,7 +48,7 @@ pub(crate) enum RbcInitialAuthenticator { Ed25519(Signer), MlDsa44(MlDsa44Signer), MlDsa65(MlDsa65Signer), - Mac, + Mac(Signer, BlsSigner), } impl RbcInitialAuthenticator { @@ -57,7 +57,7 @@ impl RbcInitialAuthenticator { Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, - Self::Mac => BlockAuthenticationScheme::MacVector, + Self::Mac(_, _) => BlockAuthenticationScheme::MacVector, } } } @@ -160,6 +160,8 @@ pub(crate) enum RbcServiceEvent { /// An irrevocable local phase statement waiting to be embedded in the /// next ordinary Starfish block. ReferenceReady(StarfishRbcReferenceV3), + EchoVoteReady(StarfishRbcEchoVoteV3), + EchoQcReady(StarfishRbcEchoQcV3), Rejected { peer: Option, error: RbcServiceError, @@ -435,7 +437,14 @@ fn validate_local_authenticator( RbcInitialAuthenticator::MlDsa65(signer) => committee .get_ml_dsa_65_public_key(own_authority) .is_some_and(|public_key| public_key == &signer.public_key()), - RbcInitialAuthenticator::Mac => committee.known_authority(own_authority), + RbcInitialAuthenticator::Mac(signer, echo_signer) => { + committee + .get_public_key(own_authority) + .is_some_and(|public_key| public_key == &signer.public_key()) + && committee + .get_bls_public_key(own_authority) + .is_some_and(|public_key| public_key == &echo_signer.public_key()) + } }; if matches { Ok(()) @@ -608,7 +617,7 @@ impl RbcServiceState { RbcInitialProof::MlDsa65(signer.sign_digest(&BlockDigest::from(digest))); self.public_initial_proposals(header, proof, transaction_data) } - RbcInitialAuthenticator::Mac => self + RbcInitialAuthenticator::Mac(_, _) => self .committee .authorities() .filter(|recipient| *recipient != self.own_authority) @@ -817,6 +826,27 @@ impl RbcServiceState { self.pending_fetches.remove(&header.reference()); let _ = self.events.send(RbcServiceEvent::Delivered(header)); } + RbcEffect::PortableEchoVote(target) => { + let RbcInitialAuthenticator::Mac(_, signer) = &self.initial_authenticator + else { + self.reject(None, RbcError::InvalidPortableEchoVote.into()); + continue; + }; + match self.kernel.portable_echo_signature_digest(target) { + Ok(digest) => { + let vote = StarfishRbcEchoVoteV3::new( + target, + self.own_authority, + signer.sign_digest(&digest), + ); + let _ = self.events.send(RbcServiceEvent::EchoVoteReady(vote)); + } + Err(error) => self.reject(None, error.into()), + } + } + RbcEffect::PortableEchoQc(qc) => { + let _ = self.events.send(RbcServiceEvent::EchoQcReady(qc)); + } } } } @@ -845,6 +875,18 @@ impl RbcServiceState { Err(error) => self.reject(Some(sender), error.into()), } } + for vote in references.echo_votes() { + match self.kernel.handle_portable_echo_vote(*vote) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(sender), error.into()), + } + } + for qc in references.echo_qcs() { + match self.kernel.handle_portable_echo_qc(qc) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(sender), error.into()), + } + } } fn note_pending_fetch(&mut self, block_ref: BlockReference, holders: AuthoritySet) { @@ -980,7 +1022,8 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, + mac_keyrings_for_test, }, starfish_rbc::RbcPhase, types::{TransactionData, VerifiedBlock}, @@ -1015,7 +1058,9 @@ mod tests { let keyrings = mac_keyrings_for_test(4); let authenticator = match scheme { BlockAuthenticationScheme::Ed25519 => RbcInitialAuthenticator::Ed25519(dummy_signer()), - BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, + BlockAuthenticationScheme::MacVector => { + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()) + } BlockAuthenticationScheme::MlDsa44 => { RbcInitialAuthenticator::MlDsa44(dummy_ml_dsa_44_signer()) } @@ -1049,7 +1094,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedCarrierDag, @@ -1070,7 +1115,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedSingleDag { @@ -1080,6 +1125,29 @@ mod tests { .unwrap() } + fn start_single_dag_fast_service() -> ( + RbcServiceHandle, + mpsc::UnboundedReceiver, + JoinHandle<()>, + ) { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + start_starfish_rbc_service_with_phase_authority( + committee, + 0, + instance(), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + 1, + Duration::from_secs(3_600), + RbcPhaseAuthorityV1::EmbeddedSingleDag { + echo_qc_fast_path: true, + }, + ) + .unwrap() + } + async fn next_event(events: &mut mpsc::UnboundedReceiver) -> RbcServiceEvent { tokio::time::timeout(Duration::from_secs(2), events.recv()) .await @@ -1223,6 +1291,44 @@ mod tests { task.await.unwrap(); } + #[tokio::test] + async fn portable_fast_path_emits_signed_echo_vote_without_phase_message() { + let (handle, mut events, task) = start_single_dag_fast_service(); + let mut header = local_header(1, 4); + header.starfish_rbc_v3 = Some(StarfishRbcFieldsV3::default()); + let canonical = handle.start_local_header(header).await.unwrap(); + let mut initials = 0; + let mut vote = None; + for _ in 0..5 { + match next_event(&mut events).await { + RbcServiceEvent::HeaderStaged(header) => { + assert_eq!(header.reference(), canonical.reference()); + } + RbcServiceEvent::Network { + message: NetworkMessage::RbcInitial(_), + .. + } => initials += 1, + RbcServiceEvent::EchoVoteReady(echo_vote) => vote = Some(echo_vote), + RbcServiceEvent::ReferenceReady(reference) + if reference.kind() == StarfishRbcReferenceKindV3::Echo => + { + panic!("portable mode emitted an unsigned ECHO reference") + } + RbcServiceEvent::Network { + message: NetworkMessage::RbcPhase(_), + .. + } => panic!("portable mode emitted a standalone phase message"), + event => panic!("unexpected portable startup event: {event:?}"), + } + } + assert_eq!(initials, 3); + let vote = vote.expect("portable mode must emit one signed ECHO vote"); + assert_eq!(vote.target(), canonical.reference()); + assert_eq!(vote.sender(), 0); + drop(handle); + task.await.unwrap(); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blocking_local_start_waits_for_kernel_selection_and_event_enqueue() { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); @@ -1620,7 +1726,7 @@ mod tests { instance(), BlockAuthenticationScheme::Ed25519, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(1), ); diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index ae1f9523..d5fd3f90 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -410,6 +410,16 @@ impl Syncer { self.try_new_block(BlockCreationReason::CertificateEvent); } + pub fn apply_starfish_rbc_echo_vote(&mut self, vote: crate::types::StarfishRbcEchoVoteV3) { + self.core.add_starfish_rbc_echo_vote(vote); + self.try_new_block(BlockCreationReason::CertificateEvent); + } + + pub fn apply_starfish_rbc_echo_qc(&mut self, qc: crate::types::StarfishRbcEchoQcV3) { + self.core.add_starfish_rbc_echo_qc(qc); + self.try_new_block(BlockCreationReason::CertificateEvent); + } + /// Sequence one exact deterministic carrier-frontier delta. In M7 this is /// the sole application-ordering authority; the legacy Starfish committer /// remains disabled in this mode. diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 6edca2ff..2afdb77a 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -138,6 +138,78 @@ pub struct StarfishRbcReferenceV3 { reference: BlockReference, } +/// A publicly verifiable ECHO vote carried by an ordinary single-DAG block. +/// +/// The signature is deliberately independent of the carrying block. A later +/// block can therefore copy a quorum of votes into a portable certificate +/// without introducing a standalone RBC phase message. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub struct StarfishRbcEchoVoteV3 { + target: BlockReference, + sender: AuthorityIndex, + signature: BlsSignatureBytes, +} + +impl StarfishRbcEchoVoteV3 { + pub fn new( + target: BlockReference, + sender: AuthorityIndex, + signature: BlsSignatureBytes, + ) -> Self { + Self { + target, + sender, + signature, + } + } + + pub fn target(self) -> BlockReference { + self.target + } + + pub fn sender(self) -> AuthorityIndex { + self.sender + } + + pub fn signature(self) -> BlsSignatureBytes { + self.signature + } +} + +/// Portable quorum certificate over exact, publicly verifiable ECHO votes. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub struct StarfishRbcEchoQcV3 { + target: BlockReference, + signers: AuthoritySet, + signature: BlsSignatureBytes, +} + +impl StarfishRbcEchoQcV3 { + pub fn new( + target: BlockReference, + signers: AuthoritySet, + signature: BlsSignatureBytes, + ) -> Self { + Self { + target, + signers, + signature, + } + } + + pub fn target(&self) -> BlockReference { + self.target + } + + pub fn signers(&self) -> AuthoritySet { + self.signers + } + + pub fn signature(&self) -> BlsSignatureBytes { + self.signature + } +} + impl StarfishRbcReferenceV3 { pub fn new(kind: StarfishRbcReferenceKindV3, reference: BlockReference) -> Self { Self { kind, reference } @@ -160,19 +232,53 @@ impl StarfishRbcReferenceV3 { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct StarfishRbcFieldsV3 { references: Vec, + #[serde(default)] + echo_votes: Vec, + #[serde(default)] + echo_qcs: Vec, } impl StarfishRbcFieldsV3 { pub fn new(mut references: Vec) -> Self { references.sort_unstable(); references.dedup(); - Self { references } + Self { + references, + echo_votes: Vec::new(), + echo_qcs: Vec::new(), + } + } + + pub fn with_portable_echo( + mut references: Vec, + mut echo_votes: Vec, + mut echo_qcs: Vec, + ) -> Self { + references.sort_unstable(); + references.dedup(); + echo_votes.sort_unstable(); + echo_votes.dedup(); + echo_qcs.sort_unstable(); + echo_qcs.dedup(); + Self { + references, + echo_votes, + echo_qcs, + } } pub fn references(&self) -> &[StarfishRbcReferenceV3] { &self.references } + pub fn echo_votes(&self) -> &[StarfishRbcEchoVoteV3] { + &self.echo_votes + } + + pub fn echo_qcs(&self) -> &[StarfishRbcEchoQcV3] { + &self.echo_qcs + } + pub(crate) fn validate_for_block( &self, committee: &Committee, @@ -181,17 +287,54 @@ impl StarfishRbcFieldsV3 { if self.references.len() > committee.len().saturating_mul(6) { return false; } + if self.echo_votes.len() > committee.len().saturating_mul(3) + || self.echo_qcs.len() > committee.len() + { + return false; + } if self.references.windows(2).any(|pair| pair[0] >= pair[1]) { return false; } let mut statements = AHashSet::new(); - self.references.iter().all(|evidence| { + let references_valid = self.references.iter().all(|evidence| { let reference = evidence.reference(); reference.round > 0 && reference.round <= block_round && committee.known_authority(reference.authority) && statements.insert((evidence.kind(), reference.authority, reference.round)) - }) + }); + if !references_valid + || self.echo_votes.windows(2).any(|pair| pair[0] >= pair[1]) + || self.echo_qcs.windows(2).any(|pair| pair[0] >= pair[1]) + { + return false; + } + let votes_valid = self.echo_votes.iter().all(|vote| { + let target = vote.target(); + target.round > 0 + && target.round <= block_round + && committee.known_authority(target.authority) + && committee.known_authority(vote.sender()) + }); + votes_valid + && self.echo_qcs.iter().all(|qc| { + let target = qc.target(); + if target.round == 0 + || target.round >= block_round + || !committee.known_authority(target.authority) + || qc.signers().is_empty() + { + return false; + } + let mut stake = 0; + for sender in qc.signers().present() { + if !committee.known_authority(sender) { + return false; + } + stake += committee.get_stake(sender).unwrap_or_default(); + } + committee.is_quorum(stake) + }) } } @@ -2359,7 +2502,9 @@ fn verify_signed_quorum( Ok(()) } -#[derive(Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, Default, Debug)] +#[derive( + Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Serialize, Deserialize, Default, Debug, +)] pub struct AuthoritySet([u64; MAX_COMMITTEE_WORDS]); pub type TimestampNs = u64; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 01847d74..8a8f14ed 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -210,9 +210,8 @@ enum Operation { /// Requires embedded RBC-DAG authority and changes the finality proof. #[clap(long, default_value_t = false)] starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only: deliver a single-DAG RBC header after quorum ECHO. - /// This preserves uniqueness but not Byzantine selective-withholding - /// totality, so it is restricted to finite benchmark runs. + /// Testbed-only: deliver a single-DAG RBC header from a portable + /// quorum of publicly signed ECHO votes carried in ordinary blocks. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. @@ -616,7 +615,7 @@ async fn local_benchmark( } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( - "Single-DAG ECHO-QC delivery: ENABLED (testbed-only; Byzantine totality not claimed)" + "Single-DAG portable ECHO-QC delivery: ENABLED (signed votes; testbed evaluation)" ); } if let Some(latency) = node_parameters.uniform_latency_ms { diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index de2e058d..233a637b 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -35,8 +35,10 @@ is one of: - `Ready(BlockReference)`. The carrying block's authenticated author is the statement sender. The list is -part of that ordinary block's content digest. Consequently no standalone phase -MAC, signature, or phase network message is needed. +part of that ordinary block's content digest. Consequently the default path +needs no standalone phase MAC, signature, or phase network message. The +portable fast path adds embedded BLS ECHO votes and an aggregate QC, but still +adds no phase message. References are sorted, duplicate-free and bounded by `6 * committee_size` per block. A sender may name at most one digest for each `(phase, target author, @@ -87,19 +89,29 @@ create a second physical round counter. - Recovery content must recompute to the exact requested `BlockReference`. - Frozen direct-RBC and carrier-DAG formats retain their old domains. -### Explicit testbed ECHO-QC fast path +### Portable ECHO-QC fast path Finite benchmarks may opt into -`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode a node delivers an -exact header as soon as it has quorum locked ECHO statements, while still -emitting the normal READY statement. Quorum intersection prevents two -different headers from both obtaining such a certificate, so agreement is -preserved. The current wire format does not carry the exact ECHO witness set, -however: Byzantine ECHO senders can selectively reveal their statements so -one honest node obtains quorum while another never can. Consequently the flag -does **not** provide the normal RBC totality guarantee and is rejected for -unbounded/production runs. The default V3 path continues to require quorum -READY. +`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode ECHO is a compact +BLS vote over the exact target reference, protocol instance and committee. +Votes ride in ordinary DAG blocks. Once quorum stake verifies, the node batch +aggregates them into one portable certificate containing the target, signer +bitmap and 48-byte aggregate signature. The QC also rides in an ordinary DAG +block; there is no standalone ECHO or READY message. + +An honest node publishes or relays the QC before its delivery effect. The +ordered Core bridge queues the QC-bearing block before applying delivery. +Quorum intersection gives uniqueness, while public verification and mandatory +relay remove the old receiver-local selective-withholding caveat: any valid QC +that reaches one honest validator can be verified and propagated by every +other validator. Missing target content still uses exact header recovery and +delivery remains fail-closed until that content is present. Invalid aggregate +batches fall back to individual vote verification so one Byzantine vote cannot +poison an otherwise valid quorum. + +The flag remains testbed-only because bounded retirement, durable pending-QC +replay and a complete asynchronous proof are still production follow-ups. The +default V3 path continues to use the conventional quorum-READY rule. ## Required validation @@ -119,10 +131,13 @@ with MAC authentication and the fixed 50 ms V3 round limiter: | --- | ---: | ---: | ---: | ---: | | `starfish-rbc-single-dag` | zero | 498.3 ms | 1,000 | 0.79 MB/s | | `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | -| V3 + flagged ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | -| V3 + flagged ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | +| V3 + old receiver-local ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | +| V3 + old receiver-local ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | +| V3 + portable aggregate ECHO-QC | zero | 405.8 ms | 1,000 | 1.06 MB/s | +| V3 + portable aggregate ECHO-QC | AWS table | 983.7 ms | 1,000 | 0.74 MB/s | | `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | All exact offered transactions committed during the bounded drain. These are -single-machine research measurements, not production claims. In particular, -both ECHO-QC rows carry the totality limitation above. +single-machine research measurements, not production claims. The two old +receiver-local rows preserve the historical totality caveat; the new portable +rows use the signed aggregate certificate described above. From 3ad9bffe0841c9de005e7da1a7036f11e10997fc Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:03:09 +0200 Subject: [PATCH 47/62] replace portable echo signatures with MAC witnesses --- crates/starfish-core/src/config.rs | 7 +- crates/starfish-core/src/core.rs | 104 ++- .../starfish-core/src/core_thread/spawned.rs | 39 +- crates/starfish-core/src/crypto.rs | 37 +- crates/starfish-core/src/metrics.rs | 142 +++- crates/starfish-core/src/net_sync.rs | 43 +- crates/starfish-core/src/network.rs | 5 + crates/starfish-core/src/starfish_rbc.rs | 669 ++++++++++-------- .../starfish-core/src/starfish_rbc_service.rs | 306 +++++--- crates/starfish-core/src/syncer.rs | 11 +- crates/starfish-core/src/types.rs | 133 +--- crates/starfish/src/main.rs | 6 +- docs/starfish-rbc-single-dag-v3.md | 57 +- 13 files changed, 954 insertions(+), 605 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 7701db4d..c99cd01a 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -94,9 +94,10 @@ pub struct NodeParameters { /// Starfish finality proof shape. #[serde(default)] pub starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed single-DAG RBC path: publicly signed ECHO votes are copied into - /// a portable quorum certificate carried by an ordinary DAG block. This - /// preserves uniqueness and totality without a standalone phase message. + /// Testbed single-DAG RBC path: exact ordinary DAG blocks carrying ECHO + /// form a signature-free MAC-witness certificate. This is a latency lower + /// bound: a Byzantine witness can selectively corrupt another receiver's + /// MAC-vector entry, so strict ECHO->READY remains the totality-safe mode. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index fe8b6fc9..cba8bf3c 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -2,7 +2,11 @@ // Modifications Copyright (c) 2025 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{collections::BTreeSet, fmt, mem, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, mem, + sync::Arc, +}; use ahash::{AHashMap, AHashSet}; use reed_solomon_simd::ReedSolomonEncoder; @@ -39,8 +43,8 @@ use crate::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishFields, Shard, - StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, StarfishRbcReferenceV3, - VerifiedBlock, + StarfishRbcEchoQcV3, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, + StarfishRbcReferenceV3, TimestampNs, VerifiedBlock, }, }; @@ -58,7 +62,12 @@ pub struct Core { /// Irrevocable local ECHO/READY statements waiting to ride on the next /// ordinary block in the single-DAG protocol. pending_starfish_rbc_references: BTreeSet, - pending_starfish_rbc_echo_votes: BTreeSet, + /// Local enqueue timestamp for each pending signature-free ECHO/READY + /// statement. This is diagnostic-only and lets the benchmark distinguish + /// network quorum time from time spent waiting for the next ordinary DAG + /// block without adding protocol messages or changing block contents. + pending_starfish_rbc_reference_queued_at_ns: + BTreeMap, pending_starfish_rbc_echo_qcs: BTreeSet, // For Byzantine node, last_own_block contains a vector of blocks last_own_block: Vec, @@ -326,7 +335,7 @@ impl Core { pending, pending_reconstructed_data: AHashMap::new(), pending_starfish_rbc_references: BTreeSet::new(), - pending_starfish_rbc_echo_votes: BTreeSet::new(), + pending_starfish_rbc_reference_queued_at_ns: BTreeMap::new(), pending_starfish_rbc_echo_qcs: BTreeSet::new(), last_own_block: vec![last_own_block], block_handler, @@ -367,23 +376,35 @@ impl Core { &self.bls_signer } - pub(crate) fn add_starfish_rbc_reference(&mut self, reference: StarfishRbcReferenceV3) { + pub(crate) fn add_starfish_rbc_reference( + &mut self, + reference: StarfishRbcReferenceV3, + target_creation_time_ns: Option, + ) { assert!( self.dag_state .consensus_protocol .is_starfish_rbc_single_dag(), "embedded RBC references require single-DAG Starfish-RBC" ); - self.pending_starfish_rbc_references.insert(reference); - } - - pub(crate) fn add_starfish_rbc_echo_vote(&mut self, vote: StarfishRbcEchoVoteV3) { - assert!( - self.dag_state - .consensus_protocol - .is_starfish_rbc_single_dag() - ); - self.pending_starfish_rbc_echo_votes.insert(vote); + if self.pending_starfish_rbc_references.insert(reference) { + if let Some(target_creation_time_ns) = target_creation_time_ns { + let now_ns = timestamp_utc() + .as_nanos() + .try_into() + .unwrap_or(TimestampNs::MAX); + self.pending_starfish_rbc_reference_queued_at_ns + .insert(reference, (now_ns, target_creation_time_ns)); + self.metrics + .observe_starfish_rbc_single_dag_phase_target_age_ns( + match reference.kind() { + StarfishRbcReferenceKindV3::Echo => "creation_to_echo_queued", + StarfishRbcReferenceKindV3::Ready => "creation_to_ready_queued", + }, + Some(now_ns.saturating_sub(target_creation_time_ns)), + ); + } + } } pub(crate) fn add_starfish_rbc_echo_qc(&mut self, qc: StarfishRbcEchoQcV3) { @@ -874,6 +895,9 @@ impl Core { }; let single_dag_rbc = protocol.is_starfish_rbc_single_dag().then(|| { let maximum = self.committee.len().saturating_mul(6); + self.metrics.set_starfish_rbc_single_dag_pending_references( + self.pending_starfish_rbc_references.len(), + ); let references: Vec<_> = self .pending_starfish_rbc_references .iter() @@ -883,28 +907,52 @@ impl Core { .collect(); for reference in &references { self.pending_starfish_rbc_references.remove(reference); + let queued_timing = self + .pending_starfish_rbc_reference_queued_at_ns + .remove(reference); + if let Some((queued_at_ns, target_creation_ns)) = queued_timing { + let now_ns = timestamp_utc() + .as_nanos() + .try_into() + .unwrap_or(TimestampNs::MAX); + self.metrics + .observe_starfish_rbc_single_dag_phase_target_age_ns( + match reference.kind() { + StarfishRbcReferenceKindV3::Echo => "creation_to_echo_embedded", + StarfishRbcReferenceKindV3::Ready => "creation_to_ready_embedded", + }, + Some(now_ns.saturating_sub(target_creation_ns)), + ); + self.metrics + .observe_starfish_rbc_single_dag_phase_target_age_ns( + match reference.kind() { + StarfishRbcReferenceKindV3::Echo => "echo_queue_dwell", + StarfishRbcReferenceKindV3::Ready => "ready_queue_dwell", + }, + Some(now_ns.saturating_sub(queued_at_ns)), + ); + } } - let echo_votes: Vec<_> = self - .pending_starfish_rbc_echo_votes - .iter() - .filter(|vote| vote.target().round <= clock_round) - .take(self.committee.len().saturating_mul(3)) - .copied() - .collect(); - for vote in &echo_votes { - self.pending_starfish_rbc_echo_votes.remove(vote); - } + self.metrics.set_starfish_rbc_single_dag_pending_references( + self.pending_starfish_rbc_references.len(), + ); let echo_qcs: Vec<_> = self .pending_starfish_rbc_echo_qcs .iter() - .filter(|qc| qc.target().round < clock_round) + .filter(|qc| { + qc.target().round < clock_round + && qc + .witnesses() + .iter() + .all(|witness| witness.round < clock_round) + }) .take(self.committee.len()) .cloned() .collect(); for qc in &echo_qcs { self.pending_starfish_rbc_echo_qcs.remove(qc); } - StarfishRbcFieldsV3::with_portable_echo(references, echo_votes, echo_qcs) + StarfishRbcFieldsV3::with_portable_echo(references, echo_qcs) }); // Create and store blocks diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 86728659..286773fa 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -81,8 +81,11 @@ enum CoreThreadCommand { ApplySailfishCertificates(Vec, oneshot::Sender<()>), /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), - ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), - ApplyStarfishRbcEchoVote(crate::types::StarfishRbcEchoVoteV3, oneshot::Sender<()>), + ApplyStarfishRbcReference( + crate::types::StarfishRbcReferenceV3, + Option, + oneshot::Sender<()>, + ), ApplyStarfishRbcEchoQc(crate::types::StarfishRbcEchoQcV3, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. ApplyStarfishRbcDagFrontier( @@ -278,25 +281,18 @@ impl, ) { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ApplyStarfishRbcReference( - reference, sender, + reference, + target_creation_time_ns, + sender, )) .await; receiver.await.expect("core thread is not expected to stop"); } - pub(crate) async fn apply_starfish_rbc_echo_vote( - &self, - vote: crate::types::StarfishRbcEchoVoteV3, - ) { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::ApplyStarfishRbcEchoVote(vote, sender)) - .await; - receiver.await.expect("core thread is not expected to stop"); - } - pub(crate) async fn apply_starfish_rbc_echo_qc(&self, qc: crate::types::StarfishRbcEchoQcV3) { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ApplyStarfishRbcEchoQc(qc, sender)) @@ -542,20 +538,17 @@ impl CoreThread { self.syncer.apply_starfish_rbc_deliveries(delivered_headers); sender.send(()).ok(); } - CoreThreadCommand::ApplyStarfishRbcReference(reference, sender) => { + CoreThreadCommand::ApplyStarfishRbcReference( + reference, + target_creation_time_ns, + sender, + ) => { metrics .core_thread_tasks_total .with_label_values(&["apply_starfish_rbc_reference"]) .inc(); - self.syncer.apply_starfish_rbc_reference(reference); - sender.send(()).ok(); - } - CoreThreadCommand::ApplyStarfishRbcEchoVote(vote, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["apply_starfish_rbc_echo_vote"]) - .inc(); - self.syncer.apply_starfish_rbc_echo_vote(vote); + self.syncer + .apply_starfish_rbc_reference(reference, target_creation_time_ns); sender.send(()).ok(); } CoreThreadCommand::ApplyStarfishRbcEchoQc(qc, sender) => { diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 27e38796..cf013da6 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -326,25 +326,19 @@ impl BlockDigest { // Preserve the frozen V3 digest exactly when the portable extension // is absent. A non-empty extension is explicitly tagged before its // length-delimited fields, so old stored/default blocks still reopen. - if !rbc.echo_votes().is_empty() || !rbc.echo_qcs().is_empty() { - hasher.update(b"PORTABLE_ECHO_QC_V1"); - let vote_len = u32::try_from(rbc.echo_votes().len()) - .expect("Starfish-RBC ECHO vote count exceeds u32"); - hasher.update(&vote_len.to_be_bytes()); - for vote in rbc.echo_votes() { - hash_reference(&mut hasher, &vote.target()); - hasher.update(&vote.sender().to_be_bytes()); - hasher.update(vote.signature().as_ref()); - } + if !rbc.echo_qcs().is_empty() { + hasher.update(b"PORTABLE_ECHO_WITNESS_QC_V1"); let qc_len = u32::try_from(rbc.echo_qcs().len()) .expect("Starfish-RBC ECHO-QC count exceeds u32"); hasher.update(&qc_len.to_be_bytes()); for qc in rbc.echo_qcs() { hash_reference(&mut hasher, &qc.target()); - for word in qc.signers().words() { - hasher.update(&word.to_be_bytes()); + let witness_len = u32::try_from(qc.witnesses().len()) + .expect("Starfish-RBC ECHO-QC witness count exceeds u32"); + hasher.update(&witness_len.to_be_bytes()); + for witness in qc.witnesses() { + hash_reference(&mut hasher, witness); } - hasher.update(qc.signature().as_ref()); } } Self(hasher.finalize().into()) @@ -1387,23 +1381,6 @@ pub fn bls_aggregate(sigs: &[&BlsSignatureBytes]) -> BlsSignatureBytes { BlsSignatureBytes(agg.to_signature().to_bytes()) } -/// Fallible counterpart used on untrusted wire votes before they have been -/// individually verified. The returned aggregate is still untrusted and must -/// be verified against the exact signer set and message. -pub fn bls_try_aggregate(sigs: &[&BlsSignatureBytes]) -> Option { - if sigs.is_empty() { - return None; - } - let parsed: Vec = sigs - .iter() - .map(|signature| bls::Signature::from_bytes(&signature.0)) - .collect::>() - .ok()?; - let references: Vec<_> = parsed.iter().collect(); - let aggregate = bls::AggregateSignature::aggregate(&references, true).ok()?; - Some(BlsSignatureBytes(aggregate.to_signature().to_bytes())) -} - /// Verify an aggregate signature against multiple public keys (all signed same /// message). #[allow(dead_code)] diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 564fdfd9..7d97702c 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -30,7 +30,7 @@ use crate::{ EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, }, stat::{DivUsize, HistogramSender, PreciseHistogram, histogram}, - types::{AuthorityIndex, format_authority_index}, + types::{AuthorityIndex, BlockReference, format_authority_index}, }; /// Metrics collected by the benchmark. @@ -150,6 +150,42 @@ const RBC_DAG_PIPELINE_LATENCY_STAGES: &[&str] = &[ RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, ]; +const RBC_SINGLE_DAG_PHASE_LATENCY_STAGES: &[&str] = &[ + "creation_to_echo_queued", + "creation_to_echo_embedded", + "echo_queue_dwell", + "creation_to_ready_queued", + "creation_to_ready_embedded", + "ready_queue_dwell", + "creation_to_delivery", +]; + +#[derive(Default)] +struct SingleDagPhaseTiming { + total_ns: [AtomicU64; 7], + samples: [AtomicU64; 7], + max_ns: [AtomicU64; 7], +} + +fn single_dag_phase_index(stage: &'static str) -> usize { + match stage { + "creation_to_echo_queued" => 0, + "creation_to_echo_embedded" => 1, + "echo_queue_dwell" => 2, + "creation_to_ready_queued" => 3, + "creation_to_ready_embedded" => 4, + "ready_queue_dwell" => 5, + "creation_to_delivery" => 6, + _ => panic!("unknown single-DAG phase timing stage {stage}"), + } +} + +/// Deterministic 1/16 diagnostic sampling by the already-random block digest. +/// The protocol path and every threshold still process all references; only +/// timing observation is sampled to keep a 40-validator local run measurable. +pub(crate) fn sample_starfish_rbc_single_dag_phase(reference: BlockReference) -> bool { + reference.digest.as_ref()[0] & 0x0f == 0 +} pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD: &str = "physical_forward"; pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD: &str = "physical_backward"; const RBC_DAG_COMMIT_DISTANCE_KINDS: &[&str] = &[ @@ -266,6 +302,12 @@ pub struct Metrics { pub network_message_bytes_sent_total: IntCounterVec, pub network_message_bytes_received_total: IntCounterVec, + /// Signature-free single-DAG RBC phase timing. Fixed lock-free slots keep + /// this diagnostic off the Prometheus label/map hot path at n=40. + starfish_rbc_single_dag_phase_timing: Arc, + starfish_rbc_single_dag_pending_references: Arc, + starfish_rbc_single_dag_pending_references_max: Arc, + // Starfish-RBC-DAG shadow instrumentation. These metrics are strictly // observational: the shadow path never feeds the authoritative DAG or // consensus state. @@ -694,6 +736,33 @@ fn format_rbc_dag_round_distance(total: u64, samples: u64, maximum: i64) -> Stri } impl Metrics { + pub(crate) fn observe_starfish_rbc_single_dag_phase_target_age_ns( + &self, + stage: &'static str, + latency_ns: Option, + ) { + let Some(latency_ns) = latency_ns else { + return; + }; + if !self.transaction_metrics_active.load(Ordering::Relaxed) { + return; + } + let index = single_dag_phase_index(stage); + self.starfish_rbc_single_dag_phase_timing.total_ns[index] + .fetch_add(latency_ns, Ordering::Relaxed); + self.starfish_rbc_single_dag_phase_timing.samples[index].fetch_add(1, Ordering::Relaxed); + self.starfish_rbc_single_dag_phase_timing.max_ns[index] + .fetch_max(latency_ns, Ordering::Relaxed); + } + + pub(crate) fn set_starfish_rbc_single_dag_pending_references(&self, pending: usize) { + let pending = u64::try_from(pending).unwrap_or(u64::MAX); + self.starfish_rbc_single_dag_pending_references + .store(pending, Ordering::Relaxed); + self.starfish_rbc_single_dag_pending_references_max + .fetch_max(pending, Ordering::Relaxed); + } + pub(crate) fn observe_starfish_rbc_dag_pipeline_latency_ns( &self, stage: &'static str, @@ -1229,6 +1298,9 @@ impl Metrics { registry, ) .unwrap(), + starfish_rbc_single_dag_phase_timing: Arc::new(SingleDagPhaseTiming::default()), + starfish_rbc_single_dag_pending_references: Arc::new(AtomicU64::new(0)), + starfish_rbc_single_dag_pending_references_max: Arc::new(AtomicU64::new(0)), starfish_rbc_dag_shadow_inputs_total: register_int_counter_vec_with_registry!( "starfish_rbc_dag_shadow_inputs_total", "Starfish-RBC-DAG shadow inputs, by bounded input kind and processing outcome", @@ -2107,6 +2179,52 @@ impl Metrics { .sum::() .as_millis() as f64 / num_validators as f64; + let single_dag_phase_latencies = RBC_SINGLE_DAG_PHASE_LATENCY_STAGES + .iter() + .enumerate() + .filter_map(|(index, stage)| { + let total_ns = metrics + .iter() + .map(|metrics| { + metrics.starfish_rbc_single_dag_phase_timing.total_ns[index] + .load(Ordering::Relaxed) + }) + .sum::(); + let samples = metrics + .iter() + .map(|metrics| { + metrics.starfish_rbc_single_dag_phase_timing.samples[index] + .load(Ordering::Relaxed) + }) + .sum::(); + if samples == 0 { + return None; + } + let maximum_ns = metrics + .iter() + .map(|metrics| { + metrics.starfish_rbc_single_dag_phase_timing.max_ns[index] + .load(Ordering::Relaxed) + }) + .max() + .unwrap_or_default(); + Some(( + *stage, + total_ns as f64 / samples as f64 / 1_000_000.0, + maximum_ns as f64 / 1_000_000.0, + samples, + )) + }) + .collect::>(); + let single_dag_pending_max = metrics + .iter() + .map(|metrics| { + metrics + .starfish_rbc_single_dag_pending_references_max + .load(Ordering::Relaxed) + }) + .max() + .unwrap_or_default(); let mut table = PrettyTable::new(); table.set_format(default_table_format()); @@ -2158,6 +2276,20 @@ impl Metrics { table.add_row(row![b->"Average TPS:", format!("{:.2} tx/s", average_tps)]); } table.add_row(row![b->"Average BPS:", format!("{:.2} blocks/s", average_bps)]); + if !single_dag_phase_latencies.is_empty() { + table.add_row(row![bH2->""]); + table.add_row(row![bH2->"Signature-Free Single-DAG RBC Phase Timing"]); + for (stage, average_ms, maximum_ms, samples) in single_dag_phase_latencies { + table.add_row(row![ + b->format!("{stage}:"), + format!("avg {average_ms:.2} ms, max {maximum_ms:.2} ms (n={samples})") + ]); + } + table.add_row(row![ + b->"Maximum pending ECHO/READY statements:", + single_dag_pending_max + ]); + } // Network metrics table.add_row(row![bH2->""]); @@ -3208,6 +3340,14 @@ mod tests { .with_label_values(&["direct_commit"]) .inc(); metrics.metrics_active.store(true, Ordering::Relaxed); + metrics + .transaction_metrics_active + .store(true, Ordering::Relaxed); + metrics.observe_starfish_rbc_single_dag_phase_target_age_ns( + "creation_to_ready_queued", + Some(11), + ); + metrics.set_starfish_rbc_single_dag_pending_references(2); metrics.observe_starfish_rbc_dag_pipeline_latency_ns( RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, 12, diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 1c95a86f..2650651b 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -43,7 +43,7 @@ use crate::{ metrics::{ Metrics, RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, - UtilizationTimerVecExt, + UtilizationTimerVecExt, sample_starfish_rbc_single_dag_phase, }, network::{BlockBatch, Connection, Network, NetworkMessage, RbcDagShadowCarrier, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, @@ -1732,6 +1732,15 @@ impl ConnectionHandler { + if let Some(ref rbc) = self.starfish_rbc_service { + if let Err(error) = rbc.header_envelope_response(self.peer_id, proposal) { + tracing::warn!( + "Failed to forward Starfish-RBC authenticated header response: {error}" + ); + } + } + } NetworkMessage::RbcDagShadowCarrier(envelope) => { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier_reliably(self.peer_id, envelope).await { @@ -3192,10 +3201,7 @@ impl NetworkSyncer BlockAuthenticationScheme::MlDsa65 => { RbcInitialAuthenticator::MlDsa65(core.get_ml_dsa_65_signer().clone()) } - BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac( - core.get_signer().clone(), - core.get_bls_signer().clone(), - ), + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, }; let (service, events, task) = start_starfish_rbc_service_with_phase_authority( committee.clone(), @@ -3542,6 +3548,17 @@ impl NetworkSyncer .await; } RbcServiceEvent::Delivered(header) => { + if sample_starfish_rbc_single_dag_phase(header.reference()) { + let now_ns = current_timestamp_ns(); + rbc_metrics.observe_starfish_rbc_single_dag_phase_target_age_ns( + "creation_to_delivery", + Some( + now_ns.saturating_sub( + header.header().meta_creation_time_ns(), + ), + ), + ); + } if let Some(ref shadow) = event_inner.starfish_rbc_dag_shadow_service { @@ -3565,16 +3582,16 @@ impl NetworkSyncer .apply_starfish_rbc_deliveries(vec![header]) .await; } - RbcServiceEvent::ReferenceReady(reference) => { - event_inner - .syncer - .apply_starfish_rbc_reference(reference) - .await; - } - RbcServiceEvent::EchoVoteReady(vote) => { + RbcServiceEvent::ReferenceReady { + reference, + target_creation_time_ns, + } => { event_inner .syncer - .apply_starfish_rbc_echo_vote(vote) + .apply_starfish_rbc_reference( + reference, + target_creation_time_ns, + ) .await; } RbcServiceEvent::EchoQcReady(qc) => { diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 63799242..1913fb55 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -295,6 +295,10 @@ pub enum NetworkMessage { /// authentication-sidecar variant. Appended after the frozen V1 message /// family so every preceding bincode enum discriminant remains stable. RbcDagShadowCarrierEnvelopeResponse(RbcDagShadowCarrierEnvelopeResponse), + /// Signature-free single-DAG recovery response carrying the exact header, + /// optional payload, and the author's complete MAC vector. Appended so + /// every preceding bincode discriminant remains stable. + RbcHeaderEnvelopeResponse(RbcHeaderProposal), } impl NetworkMessage { @@ -332,6 +336,7 @@ impl NetworkMessage { Self::RbcDagShadowCarrierEnvelopeResponse(_) => { "rbc_dag_shadow_carrier_envelope_response" } + Self::RbcHeaderEnvelopeResponse(_) => "rbc_header_envelope_response", } } } diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index feeb9446..a14f66d0 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -7,7 +7,12 @@ //! supplies content-validated headers and expands typed multicast effects into //! recipient-specific messages. -use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, + sync::Arc, +}; use ahash::{AHashMap, AHashSet}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -16,15 +21,14 @@ use crate::{ committee::{Committee, QuorumThreshold, StakeAggregator, ValidityThreshold}, crypto::{ Blake3Hasher, MacKey, MacTag, MlDsa44SignatureBytes, MlDsa65SignatureBytes, SignatureBytes, - TransactionsCommitment, bls_aggregate, bls_fast_aggregate_verify, - bls_public_keys_for_signers, bls_try_aggregate, + TransactionsCommitment, }, types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, - StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, - VerifiedBlock, compress_acknowledgments, expand_acknowledgments, + StarfishRbcEchoQcV3, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, + StarfishRbcReferenceV3, TimestampNs, TransactionData, VerifiedBlock, + compress_acknowledgments, expand_acknowledgments, }, }; @@ -33,7 +37,6 @@ const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_V1_COMMITTEE_ID"; const INITIAL_KIND: u8 = 0x00; const ECHO_KIND: u8 = 0x01; const READY_KIND: u8 = 0x02; -const PORTABLE_ECHO_KIND: u8 = 0x03; const PROTOCOL_INSTANCE_SIZE: usize = 32; const COMMITTEE_ID_SIZE: usize = 32; @@ -527,21 +530,16 @@ impl RbcCanonicalHeader { }) .ok_or(RbcError::HeaderContentTooLarge)?; let portable_echo_bytes = self.starfish_rbc_v3.as_ref().map_or(0, |rbc| { - let vote_bytes: usize = rbc - .echo_votes() - .iter() - .map(|vote| RBC_BLOCK_REFERENCE_SIZE + 2 + vote.signature().as_ref().len()) - .sum(); let qc_bytes: usize = rbc .echo_qcs() .iter() .map(|qc| { RBC_BLOCK_REFERENCE_SIZE - + qc.signers().words().len() * std::mem::size_of::() - + qc.signature().as_ref().len() + + std::mem::size_of::() + + qc.witnesses().len() * RBC_BLOCK_REFERENCE_SIZE }) .sum(); - vote_bytes.saturating_add(qc_bytes) + qc_bytes }); RBC_BLOCK_REFERENCE_SIZE .checked_mul(reference_count) @@ -601,9 +599,14 @@ pub enum RbcInitialProof { MlDsa44(MlDsa44SignatureBytes), MlDsa65(MlDsa65SignatureBytes), Mac(MacTag), + /// Complete author-generated MAC vector. Unlike a single recipient tag, + /// this proof may be relayed: each receiver verifies only its own entry. + MacVector(Vec), } -/// Direct-author Starfish-RBC header proposal carried on the wire. +/// Starfish-RBC header proposal carried on the wire. Recipient-tag proofs are +/// direct-author-only; a complete MAC-vector proof may be relayed during exact +/// witness recovery. /// /// The proof is a sidecar over the canonical header reference. It is not part /// of the content-addressed header identity. @@ -687,9 +690,11 @@ impl RbcInitialProof { BlockAuthentication::MlDsa44(signature) => Ok(Self::MlDsa44(signature.clone())), BlockAuthentication::MlDsa65(signature) => Ok(Self::MlDsa65(signature.clone())), BlockAuthentication::MacTag(tag) => Ok(Self::Mac(*tag)), - BlockAuthentication::None | BlockAuthentication::MacVector(_) => { - Err(RbcError::InvalidInitialProof) + BlockAuthentication::MacVector(tags) if !tags.is_empty() => { + Ok(Self::MacVector(tags.clone())) } + BlockAuthentication::MacVector(_) => Err(RbcError::InvalidInitialProof), + BlockAuthentication::None => Err(RbcError::InvalidInitialProof), } } } @@ -913,7 +918,6 @@ pub(crate) enum RbcEffect { holders: AuthoritySet, }, Deliver(PinnedRbcHeader), - PortableEchoVote(BlockReference), PortableEchoQc(StarfishRbcEchoQcV3), } @@ -981,7 +985,6 @@ pub(crate) enum RbcError { DuplicateAcknowledgment(BlockReference), InvalidThresholdClock, InvalidSingleDagEvidence, - InvalidPortableEchoVote, InvalidPortableEchoQc, HeaderDigestMismatch { expected: BlockDigest, @@ -1124,9 +1127,6 @@ impl fmt::Display for RbcError { Self::InvalidSingleDagEvidence => { f.write_str("Starfish-RBC V3 block carries non-canonical reference evidence") } - Self::InvalidPortableEchoVote => { - f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO vote") - } Self::InvalidPortableEchoQc => { f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO-QC") } @@ -1212,8 +1212,12 @@ struct CandidateState { ready_validity_observed: bool, ready_quorum_observed: bool, header_request_holders: AuthoritySet, - portable_echo_votes: BTreeMap, - portable_echo_votes_verified: bool, + /// Exact authenticated ordinary blocks whose authors carried + /// `ECHO(this candidate)`. These references are the signature-free QC + /// witnesses; no standalone vote object exists. + portable_echo_witnesses: BTreeMap, + pending_portable_echo_qc: Option, + portable_echo_qc_holders: AuthoritySet, portable_echo_qc_emitted: bool, portable_echo_qc_observed: bool, } @@ -1228,8 +1232,9 @@ impl CandidateState { ready_validity_observed: false, ready_quorum_observed: false, header_request_holders: AuthoritySet::default(), - portable_echo_votes: BTreeMap::new(), - portable_echo_votes_verified: false, + portable_echo_witnesses: BTreeMap::new(), + pending_portable_echo_qc: None, + portable_echo_qc_holders: AuthoritySet::default(), portable_echo_qc_emitted: false, portable_echo_qc_observed: false, } @@ -1295,7 +1300,7 @@ enum ProgressAction { NeedHeader(AuthoritySet), SendReady, Deliver, - EmitPortableEchoQc(Vec), + EmitPortableEchoQc(Vec), None, } @@ -1306,10 +1311,14 @@ pub(crate) struct StarfishRbcKernel { mac_keys: Arc>, local_round: RoundNumber, minimum_new_slot_round: RoundNumber, - /// Testbed portable fast path. Public ECHO signatures are copied into a - /// quorum certificate carried by an ordinary single-DAG block. + /// Testbed portable fast path. Exact MAC-vector-authenticated ordinary + /// blocks carrying ECHO are named by a quorum certificate in a later + /// ordinary single-DAG block. echo_qc_fast_path: bool, slots: BTreeMap>, + /// Reverse dependency index for QCs received before every exact witness + /// block is locally authenticated. Each set is committee-bounded. + portable_qc_waiters: BTreeMap>, } impl StarfishRbcKernel { @@ -1361,6 +1370,7 @@ impl StarfishRbcKernel { minimum_new_slot_round: 1, echo_qc_fast_path, slots: BTreeMap::new(), + portable_qc_waiters: BTreeMap::new(), }) } @@ -1434,7 +1444,8 @@ impl StarfishRbcKernel { header.ensure_committee(self.context.committee_id)?; let block_ref = header.reference(); self.validate_block_ref(&block_ref)?; - if direct_peer != block_ref.authority { + let portable_mac_vector = matches!(proof, RbcInitialProof::MacVector(_)); + if direct_peer != block_ref.authority && !portable_mac_vector { return Err(RbcError::InitialAuthorMismatch { expected: block_ref.authority, actual: direct_peer, @@ -1478,6 +1489,9 @@ impl StarfishRbcKernel { (BlockAuthenticationScheme::MacVector, RbcInitialProof::Mac(tag)) => { self.verify_initial_mac_tag(direct_peer, block_ref, tag)?; } + (BlockAuthenticationScheme::MacVector, RbcInitialProof::MacVector(tags)) => { + self.verify_initial_mac_vector(block_ref, tags)?; + } _ => return Err(RbcError::InitialProofSchemeMismatch), } Ok(EchoEligibleHeader { @@ -1597,7 +1611,7 @@ impl StarfishRbcKernel { ) -> Result { let pinned = self.validate_header_content(header)?; let block_ref = pinned.reference(); - if direct_peer != block_ref.authority { + if direct_peer != block_ref.authority && !matches!(proof, RbcInitialProof::MacVector(_)) { return Err(RbcError::InitialAuthorMismatch { expected: block_ref.authority, actual: direct_peer, @@ -1679,15 +1693,12 @@ impl StarfishRbcKernel { .echoes .add(own_authority, &committee); - let mut effects = vec![if self.echo_qc_fast_path { - RbcEffect::PortableEchoVote(block_ref) - } else { - RbcEffect::MulticastPhase { - phase: RbcPhase::Echo, - block_ref, - } + let mut effects = vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, }]; effects.extend(self.drive(block_ref)); + effects.extend(self.retry_portable_qcs_waiting_on(block_ref)); Ok(effects) } @@ -1723,21 +1734,43 @@ impl StarfishRbcKernel { pub(crate) fn handle_embedded_reference( &mut self, authenticated_sender: AuthorityIndex, + enclosing_block: BlockReference, evidence: StarfishRbcReferenceV3, ) -> Result, RbcError> { if !self.committee.known_authority(authenticated_sender) { return Err(RbcError::UnknownAuthority(authenticated_sender)); } + if enclosing_block.authority != authenticated_sender { + return Err(RbcError::InvalidPortableEchoQc); + } let block_ref = evidence.reference(); self.validate_block_ref(&block_ref)?; let phase = match evidence.kind() { StarfishRbcReferenceKindV3::Echo => RbcPhase::Echo, StarfishRbcReferenceKindV3::Ready => RbcPhase::Ready, }; + let echo_qc_fast_path = self.echo_qc_fast_path; let committee = Arc::clone(&self.committee); let slot = self.slot_mut(block_ref); + if phase == RbcPhase::Echo && echo_qc_fast_path { + let candidate = slot + .candidates + .entry(block_ref) + .or_insert_with(CandidateState::new); + match candidate.portable_echo_witnesses.get(&authenticated_sender) { + Some(existing) if *existing != enclosing_block => { + return Err(RbcError::InvalidPortableEchoQc); + } + Some(_) => {} + None => { + candidate + .portable_echo_witnesses + .insert(authenticated_sender, enclosing_block); + } + } + } if !slot.record_phase_sender(phase, authenticated_sender, block_ref) { - return Ok(Vec::new()); + return Ok(self.drive(block_ref)); } let candidate = slot .candidates @@ -1754,127 +1787,17 @@ impl StarfishRbcKernel { Ok(self.drive(block_ref)) } - /// Digest signed by an ECHO sender for the portable single-DAG fast path. - /// The protocol instance and committee identifier prevent cross-run reuse. - pub(crate) fn portable_echo_signature_digest( - &self, - target: BlockReference, - ) -> Result<[u8; 32], RbcError> { - self.validate_block_ref(&target)?; - Ok(blake3::hash(&encode_base_statement( - &self.context, - PORTABLE_ECHO_KIND, - &target, - )) - .into()) - } - - pub(crate) fn handle_portable_echo_vote( - &mut self, - vote: StarfishRbcEchoVoteV3, - ) -> Result, RbcError> { - if !self.echo_qc_fast_path { - return Err(RbcError::InvalidPortableEchoVote); - } - self.validate_block_ref(&vote.target())?; - if self - .candidate(&vote.target()) - .is_some_and(|candidate| candidate.portable_echo_votes_verified) - { - return Ok(Vec::new()); - } - let candidate = self.candidate_mut(vote.target()); - match candidate.portable_echo_votes.get(&vote.sender()) { - Some(existing) if existing != &vote => return Err(RbcError::InvalidPortableEchoVote), - Some(_) => return Ok(Vec::new()), - None => { - candidate.portable_echo_votes.insert(vote.sender(), vote); - } - } - self.verify_portable_echo_vote_batch(vote.target())?; - Ok(self.drive(vote.target())) - } - - fn verify_portable_echo_vote_batch(&mut self, target: BlockReference) -> Result<(), RbcError> { - let votes: Vec<_> = self - .candidate(&target) - .into_iter() - .flat_map(|candidate| candidate.portable_echo_votes.values().copied()) - .collect(); - let stake: Stake = votes - .iter() - .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) - .sum(); - if !self.committee.is_quorum(stake) { - return Ok(()); - } - let digest = self.portable_echo_signature_digest(target)?; - let batch_valid = |votes: &[StarfishRbcEchoVoteV3]| { - let mut signers = AuthoritySet::default(); - for vote in votes { - signers.insert(vote.sender()); - } - let signatures: Vec<_> = votes.iter().map(|vote| vote.signature()).collect(); - let signature_refs: Vec<_> = signatures.iter().collect(); - let Some(aggregate) = bls_try_aggregate(&signature_refs) else { - return false; - }; - let Some(public_keys) = bls_public_keys_for_signers(&self.committee, signers) else { - return false; - }; - bls_fast_aggregate_verify(&digest, &aggregate, &public_keys) - }; - let valid_votes = if batch_valid(&votes) { - votes - } else { - votes - .into_iter() - .filter(|vote| { - self.committee - .get_bls_public_key(vote.sender()) - .is_some_and(|public_key| { - public_key - .verify_trusted(&digest, &vote.signature()) - .is_ok() - }) - }) - .collect() - }; - let valid_stake: Stake = valid_votes - .iter() - .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) - .sum(); - let committee = Arc::clone(&self.committee); - let slot = self.slot_mut(target); - if committee.is_quorum(valid_stake) { - for vote in &valid_votes { - slot.record_phase_sender(RbcPhase::Echo, vote.sender(), target); - } - } - let candidate = slot - .candidates - .entry(target) - .or_insert_with(CandidateState::new); - candidate.portable_echo_votes = valid_votes - .iter() - .map(|vote| (vote.sender(), *vote)) - .collect(); - if committee.is_quorum(valid_stake) { - candidate.portable_echo_votes_verified = true; - for vote in valid_votes { - candidate.echoes.add(vote.sender(), &committee); - } - } - Ok(()) - } - pub(crate) fn handle_portable_echo_qc( &mut self, + authenticated_sender: AuthorityIndex, qc: &StarfishRbcEchoQcV3, ) -> Result, RbcError> { if !self.echo_qc_fast_path { return Err(RbcError::InvalidPortableEchoQc); } + if !self.committee.known_authority(authenticated_sender) { + return Err(RbcError::UnknownAuthority(authenticated_sender)); + } self.validate_block_ref(&qc.target())?; if self .candidate(&qc.target()) @@ -1882,45 +1805,139 @@ impl StarfishRbcKernel { { return Ok(self.drive(qc.target())); } + self.validate_portable_echo_qc_shape(qc)?; + let candidate = self.candidate_mut(qc.target()); + candidate + .portable_echo_qc_holders + .insert(authenticated_sender); + match candidate.pending_portable_echo_qc.as_ref() { + Some(existing) if existing <= qc => {} + _ => candidate.pending_portable_echo_qc = Some(qc.clone()), + } + Ok(self.try_complete_portable_echo_qc(qc.target())) + } + + fn validate_portable_echo_qc_shape(&self, qc: &StarfishRbcEchoQcV3) -> Result<(), RbcError> { + if qc.witnesses().is_empty() + || qc.witnesses().len() > self.committee.len() + || qc.witnesses().windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(RbcError::InvalidPortableEchoQc); + } + let mut authors = AuthoritySet::default(); let mut stake = 0; - for sender in qc.signers().present() { + for witness in qc.witnesses() { + self.validate_block_ref(witness)?; + if witness.round <= qc.target().round || authors.contains(witness.authority) { + return Err(RbcError::InvalidPortableEchoQc); + } + authors.insert(witness.authority); stake += self .committee - .get_stake(sender) + .get_stake(witness.authority) .ok_or(RbcError::InvalidPortableEchoQc)?; } if !self.committee.is_quorum(stake) { return Err(RbcError::InvalidPortableEchoQc); } - let public_keys = bls_public_keys_for_signers(&self.committee, qc.signers()) - .ok_or(RbcError::InvalidPortableEchoQc)?; - if !bls_fast_aggregate_verify( - &self.portable_echo_signature_digest(qc.target())?, - &qc.signature(), - &public_keys, - ) { - return Err(RbcError::InvalidPortableEchoQc); + Ok(()) + } + + fn authenticated_witness_carries_echo( + &self, + witness: BlockReference, + target: BlockReference, + ) -> Option { + let slot = self.slot(&witness)?; + if slot.echoed != Some(witness) { + return None; } - let candidate = self.candidate_mut(qc.target()); + let header = slot.candidates.get(&witness)?.header.as_ref()?; + Some(header.header().starfish_rbc_v3().is_some_and(|fields| { + fields + .references() + .binary_search(&StarfishRbcReferenceV3::new( + StarfishRbcReferenceKindV3::Echo, + target, + )) + .is_ok() + })) + } + + fn try_complete_portable_echo_qc(&mut self, target: BlockReference) -> Vec { + let Some((qc, holders)) = self.candidate(&target).and_then(|candidate| { + candidate + .pending_portable_echo_qc + .clone() + .map(|qc| (qc, candidate.portable_echo_qc_holders)) + }) else { + return Vec::new(); + }; + + let mut missing = Vec::new(); + for witness in qc.witnesses() { + match self.authenticated_witness_carries_echo(*witness, target) { + Some(true) => {} + Some(false) => { + self.candidate_mut(target).pending_portable_echo_qc = None; + return Vec::new(); + } + None => missing.push(*witness), + } + } + if !missing.is_empty() { + for witness in &missing { + self.portable_qc_waiters + .entry(*witness) + .or_default() + .insert(target); + } + return missing + .into_iter() + .map(|block_ref| RbcEffect::NeedHeader { block_ref, holders }) + .collect(); + } + + for witness in qc.witnesses() { + if let Some(waiters) = self.portable_qc_waiters.get_mut(witness) { + waiters.remove(&target); + if waiters.is_empty() { + self.portable_qc_waiters.remove(witness); + } + } + } + let candidate = self.candidate_mut(target); + candidate.pending_portable_echo_qc = None; candidate.portable_echo_qc_observed = true; let relay = if candidate.portable_echo_qc_emitted { None } else { candidate.portable_echo_qc_emitted = true; - Some(RbcEffect::PortableEchoQc(qc.clone())) + Some(RbcEffect::PortableEchoQc(qc)) }; let mut effects = relay.into_iter().collect::>(); - effects.extend(self.drive(qc.target())); + effects.extend(self.drive(target)); if self - .candidate(&qc.target()) + .candidate(&target) .is_some_and(|candidate| candidate.header.is_none()) { effects.push(RbcEffect::NeedHeader { - block_ref: qc.target(), - holders: qc.signers(), + block_ref: target, + holders, }); } - Ok(effects) + effects + } + + fn retry_portable_qcs_waiting_on(&mut self, witness: BlockReference) -> Vec { + let targets = self + .portable_qc_waiters + .remove(&witness) + .unwrap_or_default(); + targets + .into_iter() + .flat_map(|target| self.try_complete_portable_echo_qc(target)) + .collect() } /// Materialize one recipient-specific message for an untagged multicast @@ -2020,6 +2037,26 @@ impl StarfishRbcKernel { self.make_initial_mac_tag_for_reference(block_ref, recipient) } + pub(crate) fn make_local_initial_mac_vector( + &self, + local: &RbcLocalInitial, + ) -> Result, RbcError> { + let block_ref = self.ensure_local_initial(local)?; + if self.context.initial_authentication != BlockAuthenticationScheme::MacVector { + return Err(RbcError::InitialMacRequiresMacAuthentication); + } + self.committee + .authorities() + .map(|recipient| { + if recipient == self.own_authority { + Ok(MacTag::from_bytes([0; crate::crypto::MAC_TAG_SIZE])) + } else { + self.make_initial_mac_tag_for_reference(block_ref, recipient) + } + }) + .collect() + } + fn make_initial_mac_tag_for_reference( &self, block_ref: BlockReference, @@ -2085,6 +2122,35 @@ impl StarfishRbcKernel { Ok(()) } + pub(crate) fn verify_initial_mac_vector( + &self, + block_ref: BlockReference, + tags: &[MacTag], + ) -> Result<(), RbcError> { + if self.context.initial_authentication != BlockAuthenticationScheme::MacVector { + return Err(RbcError::InitialMacRequiresMacAuthentication); + } + self.validate_block_ref(&block_ref)?; + if tags.len() != self.committee.len() { + return Err(RbcError::InvalidInitialProof); + } + if block_ref.authority == self.own_authority { + return Err(RbcError::LoopbackPhase); + } + let statement = encode_mac_statement( + &self.context, + INITIAL_KIND, + &block_ref, + block_ref.authority, + self.own_authority, + ); + let expected = self.mac_keys[block_ref.authority as usize].compute_rbc_tag(&statement); + if tags[self.own_authority as usize] != expected { + return Err(RbcError::InvalidInitialTag); + } + Ok(()) + } + #[allow(dead_code)] pub(crate) fn header_holders(&self, block_ref: &BlockReference) -> AuthoritySet { self.candidate(block_ref) @@ -2260,15 +2326,14 @@ impl StarfishRbcKernel { let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; - let portable_vote_stake = candidate - .portable_echo_votes + let portable_witness_stake = candidate + .portable_echo_witnesses .keys() .map(|sender| committee.get_stake(*sender).unwrap_or_default()) .sum::(); let portable_qc_ready = echo_qc_fast_path && !candidate.portable_echo_qc_emitted - && candidate.portable_echo_votes_verified - && committee.is_quorum(portable_vote_stake); + && committee.is_quorum(portable_witness_stake); let blocked_on_header = candidate.header.is_none() && ((can_send_ready && ready_trigger) || (can_deliver @@ -2277,7 +2342,11 @@ impl StarfishRbcKernel { let holders = candidate.holders(); if portable_qc_ready { ProgressAction::EmitPortableEchoQc( - candidate.portable_echo_votes.values().copied().collect(), + candidate + .portable_echo_witnesses + .values() + .copied() + .collect(), ) } else if blocked_on_header && holders != candidate.header_request_holders { candidate.header_request_holders = holders; @@ -2335,7 +2404,7 @@ impl StarfishRbcKernel { effects.push(RbcEffect::Deliver(header)); } } - ProgressAction::EmitPortableEchoQc(votes) => { + ProgressAction::EmitPortableEchoQc(witnesses) => { let candidate = self.candidate_mut(block_ref); if !candidate.portable_echo_qc_emitted { candidate.portable_echo_qc_emitted = true; @@ -2344,17 +2413,8 @@ impl StarfishRbcKernel { // creation/dissemination of the QC-bearing block, so // delivery cannot overtake portable publication. candidate.portable_echo_qc_observed = true; - let mut signers = AuthoritySet::default(); - for vote in &votes { - signers.insert(vote.sender()); - } - let signatures: Vec<_> = - votes.iter().map(|vote| vote.signature()).collect(); - let signature_refs: Vec<_> = signatures.iter().collect(); effects.push(RbcEffect::PortableEchoQc(StarfishRbcEchoQcV3::new( - block_ref, - signers, - bls_aggregate(&signature_refs), + block_ref, witnesses, ))); } } @@ -2467,8 +2527,7 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, - mac_keyrings_for_test, + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, types::{BlockDigest, BlockReference}, }; @@ -2539,11 +2598,13 @@ mod tests { let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); assert!( receiver - .handle_embedded_reference(1, echo) + .handle_embedded_reference(1, block(1, 2, 0x81), echo) .unwrap() .is_empty() ); - let effects = receiver.handle_embedded_reference(2, echo).unwrap(); + let effects = receiver + .handle_embedded_reference(2, block(2, 2, 0x82), echo) + .unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::MulticastPhase { @@ -2560,11 +2621,13 @@ mod tests { let ready = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Ready, target); assert!( receiver - .handle_embedded_reference(1, ready) + .handle_embedded_reference(1, block(1, 3, 0x91), ready) .unwrap() .is_empty() ); - let effects = receiver.handle_embedded_reference(2, ready).unwrap(); + let effects = receiver + .handle_embedded_reference(2, block(2, 3, 0x92), ready) + .unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::Deliver(header) if header.reference() == target @@ -2572,7 +2635,7 @@ mod tests { } #[test] - fn portable_echo_qc_fast_path_requires_and_accepts_exact_signed_quorum() { + fn portable_echo_qc_fast_path_uses_exact_authenticated_dag_witnesses() { let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(committee.len()); let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( @@ -2591,21 +2654,32 @@ mod tests { .unwrap(); receiver.authorize_echo(target).unwrap(); - let digest = receiver.portable_echo_signature_digest(target).unwrap(); - let signer = dummy_bls_signer(); - let votes: Vec<_> = (0..3) - .map(|sender| StarfishRbcEchoVoteV3::new(target, sender, signer.sign_digest(&digest))) + let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); + let witnesses: Vec<_> = (0..3) + .map(|sender| block(sender, 2, 0x80 + sender as u8)) .collect(); - for vote in &votes[..2] { - assert!( - !receiver - .handle_portable_echo_vote(*vote) - .unwrap() - .iter() - .any(|effect| matches!(effect, RbcEffect::Deliver(_))) - ); + let mut effects = Vec::new(); + for witness in &witnesses { + let header = PinnedRbcHeader { + header: Arc::new(RbcCanonicalHeader { + reference: *witness, + block_references: Vec::new(), + acknowledgments: RbcAckFields { + intersection: Some(0), + extra_references: Vec::new(), + }, + meta_creation_time_ns: 0, + transactions_commitment: TransactionsCommitment::default(), + starfish_rbc_v3: Some(StarfishRbcFieldsV3::new(vec![echo])), + }), + committee_id: receiver.context.committee_id, + }; + receiver.note_header_available(header).unwrap(); + receiver.authorize_echo(*witness).unwrap(); + effects = receiver + .handle_embedded_reference(witness.authority, *witness, echo) + .unwrap(); } - let effects = receiver.handle_portable_echo_vote(votes[2]).unwrap(); assert!(!effects.iter().any(|effect| matches!( effect, @@ -2620,7 +2694,8 @@ mod tests { RbcEffect::PortableEchoQc(qc) => Some(qc.clone()), _ => None, }) - .expect("signed quorum must emit a portable ECHO-QC"); + .expect("exact witness quorum must emit a portable ECHO-QC"); + assert_eq!(qc.witnesses(), witnesses); let qc_position = effects .iter() .position(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) @@ -2635,7 +2710,7 @@ mod tests { }) .expect("delivery follows portable publication"); assert!(qc_position < delivery_position); - assert!(receiver.handle_portable_echo_qc(&qc).unwrap().is_empty()); + assert!(receiver.handle_portable_echo_qc(1, &qc).unwrap().is_empty()); let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(committee.len()); @@ -2649,77 +2724,52 @@ mod tests { true, ) .unwrap(); - let missing = late_receiver.handle_portable_echo_qc(&qc).unwrap(); - assert!(missing.iter().any(|effect| matches!( + late_receiver + .note_header_available(pinned_header_for_context(late_receiver.context, target)) + .unwrap(); + let missing = late_receiver.handle_portable_echo_qc(1, &qc).unwrap(); + assert_eq!( + missing + .iter() + .filter(|effect| matches!(effect, RbcEffect::NeedHeader { .. })) + .count(), + witnesses.len() + ); + let mut effects = Vec::new(); + for witness in &witnesses { + let header = PinnedRbcHeader { + header: Arc::new(RbcCanonicalHeader { + reference: *witness, + block_references: Vec::new(), + acknowledgments: RbcAckFields { + intersection: Some(0), + extra_references: Vec::new(), + }, + meta_creation_time_ns: 0, + transactions_commitment: TransactionsCommitment::default(), + starfish_rbc_v3: Some(StarfishRbcFieldsV3::new(vec![echo])), + }), + committee_id: late_receiver.context.committee_id, + }; + late_receiver.note_header_available(header).unwrap(); + effects = late_receiver.authorize_echo(*witness).unwrap(); + } + assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::PortableEchoQc(relayed) if relayed == &qc ))); - assert!(missing.iter().any(|effect| matches!( - effect, - RbcEffect::NeedHeader { block_ref, .. } if *block_ref == target - ))); - let effects = late_receiver - .note_header_available(pinned_header_for_context(late_receiver.context, target)) - .unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::Deliver(header) if header.reference() == target ))); let other_target = block(3, 1, 0x73); - let forged = StarfishRbcEchoQcV3::new(other_target, qc.signers(), qc.signature()); - assert_eq!( - receiver.handle_portable_echo_qc(&forged), - Err(RbcError::InvalidPortableEchoQc) - ); - } - - #[test] - fn invalid_echo_vote_cannot_poison_a_later_portable_quorum() { - let committee = Committee::new_test(vec![1; 4]); - let keyrings = mac_keyrings_for_test(committee.len()); - let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( - committee, - 0, - instance(TEST_INSTANCE_BYTE), - BlockAuthenticationScheme::MacVector, - Arc::new(keyrings[0].clone()), - 1, - true, - ) - .unwrap(); - let target = block(3, 1, 0x74); - receiver - .note_header_available(pinned_header_for_context(receiver.context, target)) - .unwrap(); - let digest = receiver.portable_echo_signature_digest(target).unwrap(); - let signer = dummy_bls_signer(); - let wrong_signature = signer.sign_digest(&[0xFF; 32]); - for (sender, signature) in [ - (0, signer.sign_digest(&digest)), - (1, wrong_signature), - (2, signer.sign_digest(&digest)), - ] { - let effects = receiver - .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new(target, sender, signature)) - .unwrap(); - assert!( - !effects - .iter() - .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) - ); - } - let effects = receiver - .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new( - target, - 3, - signer.sign_digest(&digest), - )) - .unwrap(); + let forged = StarfishRbcEchoQcV3::new(other_target, witnesses); + let effects = receiver.handle_portable_echo_qc(1, &forged).unwrap(); assert!( - effects + !effects .iter() - .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) + .any(|effect| matches!(effect, RbcEffect::Deliver(_))) ); } @@ -2911,7 +2961,7 @@ mod tests { ); deliveries[owner as usize].push(header.reference()); } - RbcEffect::PortableEchoVote(_) | RbcEffect::PortableEchoQc(_) => { + RbcEffect::PortableEchoQc(_) => { panic!("portable ECHO effects require the single-DAG test harness") } } @@ -3062,19 +3112,14 @@ mod tests { } #[test] - fn single_dag_digest_binds_portable_echo_votes_and_certificate() { + fn single_dag_digest_binds_portable_echo_witness_certificate() { let target = block(0, 4, 0x43); - let signature = dummy_bls_signer().sign_digest(&[0x44; 32]); - let vote = StarfishRbcEchoVoteV3::new(target, 1, signature); - let mut signers = AuthoritySet::default(); - signers.insert(0); - signers.insert(1); - signers.insert(2); - let signatures = [&signature, &signature, &signature]; - let qc = StarfishRbcEchoQcV3::new(target, signers, bls_aggregate(&signatures)); + let qc = StarfishRbcEchoQcV3::new( + target, + vec![block(0, 5, 0x44), block(1, 5, 0x45), block(2, 5, 0x46)], + ); let empty = StarfishRbcFieldsV3::default(); - let with_vote = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), vec![vote], Vec::new()); - let with_qc = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), Vec::new(), vec![qc]); + let with_qc = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), vec![qc]); let digest = |fields: &StarfishRbcFieldsV3| { BlockDigest::new_starfish_rbc_single_dag_header( 3, @@ -3086,9 +3131,7 @@ mod tests { fields, ) }; - assert_ne!(digest(&empty), digest(&with_vote)); assert_ne!(digest(&empty), digest(&with_qc)); - assert_ne!(digest(&with_vote), digest(&with_qc)); } #[test] @@ -3972,6 +4015,66 @@ mod tests { )); } + #[test] + fn complete_initial_mac_vector_is_relayable_and_receiver_verifiable() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(4); + let template = valid_canonical_header(0, 7, 0x45); + let mut author = kernel( + Arc::clone(&committee), + &keyrings, + 0, + BlockAuthenticationScheme::MacVector, + ); + let local = author + .start_local_initial_header( + template.reference().round, + template.block_references().to_vec(), + template.acknowledgment_references(), + template.meta_creation_time_ns(), + template.transactions_commitment(), + ) + .unwrap(); + let canonical = local.header().clone(); + let vector = author.make_local_initial_mac_vector(&local).unwrap(); + assert_eq!(vector.len(), committee.len()); + + let mut receiver = kernel( + Arc::clone(&committee), + &keyrings, + 2, + BlockAuthenticationScheme::MacVector, + ); + assert!(matches!( + receiver + .accept_direct_initial_header( + 1, + canonical.clone(), + &RbcInitialProof::MacVector(vector.clone()), + ) + .unwrap(), + RbcInitialHeaderOutcome::Authenticated { .. } + )); + + let mut poisoned = vector; + poisoned[2] = MacTag::from_bytes([0; crate::crypto::MAC_TAG_SIZE]); + let mut other_receiver = kernel( + committee, + &keyrings, + 2, + BlockAuthenticationScheme::MacVector, + ); + assert!(matches!( + other_receiver + .accept_direct_initial_header(1, canonical, &RbcInitialProof::MacVector(poisoned),) + .unwrap(), + RbcInitialHeaderOutcome::StagedUnauthenticated { + error: RbcError::InvalidInitialTag, + .. + } + )); + } + #[test] fn signature_digest_binds_context_scheme_and_reference() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 90941860..c6b234ef 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -25,7 +25,8 @@ use tokio::{ use crate::{ committee::Committee, - crypto::{BlsSigner, MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + metrics::sample_starfish_rbc_single_dag_phase, network::NetworkMessage, starfish_rbc::{ PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, @@ -34,8 +35,8 @@ use crate::{ }, types::{ AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - RoundNumber, StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, - StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, + RoundNumber, StarfishRbcEchoQcV3, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, + StarfishRbcReferenceV3, TimestampNs, TransactionData, }, }; @@ -48,7 +49,9 @@ pub(crate) enum RbcInitialAuthenticator { Ed25519(Signer), MlDsa44(MlDsa44Signer), MlDsa65(MlDsa65Signer), - Mac(Signer, BlsSigner), + /// Signature-free initial authentication. The kernel derives the complete + /// receiver-verifiable MAC vector from its pairwise keyring. + Mac, } impl RbcInitialAuthenticator { @@ -57,7 +60,7 @@ impl RbcInitialAuthenticator { Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, - Self::Mac(_, _) => BlockAuthenticationScheme::MacVector, + Self::Mac => BlockAuthenticationScheme::MacVector, } } } @@ -159,8 +162,10 @@ pub(crate) enum RbcServiceEvent { Delivered(PinnedRbcHeader), /// An irrevocable local phase statement waiting to be embedded in the /// next ordinary Starfish block. - ReferenceReady(StarfishRbcReferenceV3), - EchoVoteReady(StarfishRbcEchoVoteV3), + ReferenceReady { + reference: StarfishRbcReferenceV3, + target_creation_time_ns: Option, + }, EchoQcReady(StarfishRbcEchoQcV3), Rejected { peer: Option, @@ -190,6 +195,10 @@ enum RbcServiceMessage { peer: AuthorityIndex, header: RbcCanonicalHeader, }, + HeaderEnvelopeResponse { + peer: AuthorityIndex, + proposal: RbcHeaderProposal, + }, PeerConnected(AuthorityIndex), PeerDisconnected(AuthorityIndex), #[allow(dead_code)] @@ -291,6 +300,14 @@ impl RbcServiceHandle { self.send(RbcServiceMessage::HeaderResponse { peer, header }) } + pub(crate) fn header_envelope_response( + &self, + peer: AuthorityIndex, + proposal: RbcHeaderProposal, + ) -> Result<(), RbcServiceError> { + self.send(RbcServiceMessage::HeaderEnvelopeResponse { peer, proposal }) + } + pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), RbcServiceError> { self.send(RbcServiceMessage::PeerConnected(peer)) } @@ -414,7 +431,9 @@ pub(crate) fn start_starfish_rbc_service_with_phase_authority( connected_peers: AuthoritySet::default(), pending_fetches: AHashMap::new(), staged_notifications: AHashSet::new(), + header_creation_times: AHashMap::new(), retained_initials: BTreeMap::new(), + retained_envelopes: AHashMap::new(), retained_phases: BTreeSet::new(), phase_authority, }; @@ -437,14 +456,7 @@ fn validate_local_authenticator( RbcInitialAuthenticator::MlDsa65(signer) => committee .get_ml_dsa_65_public_key(own_authority) .is_some_and(|public_key| public_key == &signer.public_key()), - RbcInitialAuthenticator::Mac(signer, echo_signer) => { - committee - .get_public_key(own_authority) - .is_some_and(|public_key| public_key == &signer.public_key()) - && committee - .get_bls_public_key(own_authority) - .is_some_and(|public_key| public_key == &echo_signer.public_key()) - } + RbcInitialAuthenticator::Mac => committee.known_authority(own_authority), }; if matches { Ok(()) @@ -491,9 +503,16 @@ struct RbcServiceState { connected_peers: AuthoritySet, pending_fetches: AHashMap, staged_notifications: AHashSet, + /// Diagnostic timestamp cache populated at the existing header-staging + /// boundary. A direct hash lookup avoids re-walking the kernel's nested + /// historical slot maps when ECHO or READY becomes locally eligible. + header_creation_times: AHashMap, /// Recipient-specialized local proposals retained for replay after a /// connection is replaced. Version one keeps these for the run. retained_initials: BTreeMap<(BlockReference, AuthorityIndex), RbcHeaderProposal>, + /// Authenticated full-vector proposal retained once per exact header for + /// signature-free witness recovery and relay. + retained_envelopes: AHashMap, /// Authorized local phase intents. Tags are rematerialized for the peer /// on replay rather than retaining or cloning a tagged wire message. retained_phases: BTreeSet<(BlockReference, RbcPhase)>, @@ -528,6 +547,9 @@ impl RbcServiceState { RbcServiceMessage::HeaderResponse { peer, header } => { self.accept_header_response(peer, header); } + RbcServiceMessage::HeaderEnvelopeResponse { peer, proposal } => { + self.accept_header_envelope_response(peer, proposal); + } RbcServiceMessage::PeerConnected(peer) => self.peer_connected(peer), RbcServiceMessage::PeerDisconnected(peer) => self.peer_disconnected(peer), RbcServiceMessage::AdvanceLocalRound { round, reply } => { @@ -571,6 +593,13 @@ impl RbcServiceState { let embedded_references = canonical.starfish_rbc_v3().cloned(); let transaction_data = transaction_data.map(Arc::new); let proposals = self.make_initial_proposals(&local, transaction_data); + if let Some((_, proposal)) = proposals + .first() + .filter(|(_, proposal)| matches!(proposal.proof(), RbcInitialProof::MacVector(_))) + { + self.retained_envelopes + .insert(canonical.reference(), proposal.clone()); + } let (pinned, effects) = local.into_parts(); self.notify_header_staged(pinned); @@ -580,7 +609,11 @@ impl RbcServiceState { self.send_network(recipient, NetworkMessage::RbcInitial(proposal)); } self.process_effects(effects); - self.process_embedded_references(self.own_authority, embedded_references); + self.process_embedded_references( + self.own_authority, + canonical.reference(), + embedded_references, + ); Ok(canonical) } @@ -617,25 +650,52 @@ impl RbcServiceState { RbcInitialProof::MlDsa65(signer.sign_digest(&BlockDigest::from(digest))); self.public_initial_proposals(header, proof, transaction_data) } - RbcInitialAuthenticator::Mac(_, _) => self - .committee - .authorities() - .filter(|recipient| *recipient != self.own_authority) - .map(|recipient| { - let tag = self + RbcInitialAuthenticator::Mac => { + if matches!( + self.phase_authority, + RbcPhaseAuthorityV1::EmbeddedSingleDag { + echo_qc_fast_path: true + } + ) { + let vector = self .kernel - .make_local_initial_mac_tag(local, recipient) + .make_local_initial_mac_vector(local) .expect("local RBC handle must remain selected"); - ( - recipient, - RbcHeaderProposal::with_transaction_data( - header.clone(), - RbcInitialProof::Mac(tag), - transaction_data.clone(), - ), - ) - }) - .collect(), + self.committee + .authorities() + .filter(|recipient| *recipient != self.own_authority) + .map(|recipient| { + ( + recipient, + RbcHeaderProposal::with_transaction_data( + header.clone(), + RbcInitialProof::MacVector(vector.clone()), + transaction_data.clone(), + ), + ) + }) + .collect() + } else { + self.committee + .authorities() + .filter(|recipient| *recipient != self.own_authority) + .map(|recipient| { + let tag = self + .kernel + .make_local_initial_mac_tag(local, recipient) + .expect("local RBC handle must remain selected"); + ( + recipient, + RbcHeaderProposal::with_transaction_data( + header.clone(), + RbcInitialProof::Mac(tag), + transaction_data.clone(), + ), + ) + }) + .collect() + } + } } } @@ -662,6 +722,7 @@ impl RbcServiceState { } fn accept_direct_initial(&mut self, peer: AuthorityIndex, proposal: RbcHeaderProposal) { + let retained = proposal.clone(); let (header, proof, transaction_data) = proposal.into_parts(); let block_ref = header.reference(); let embedded_references = header.starfish_rbc_v3().cloned(); @@ -670,14 +731,32 @@ impl RbcServiceState { .accept_direct_initial_header(peer, header, &proof) { Ok(RbcInitialHeaderOutcome::Authenticated { effects }) => { + if matches!(retained.proof(), RbcInitialProof::MacVector(_)) { + self.retained_envelopes.insert(block_ref, retained); + } let pinned = self.finish_header_staging(block_ref, Some(peer)); self.notify_transaction_payload(peer, pinned, transaction_data); self.process_effects(effects); - self.process_embedded_references(peer, embedded_references); + // A full MAC vector authenticates the canonical header's + // author even when `peer` is merely recovering/relaying it. + self.process_embedded_references( + block_ref.authority, + block_ref, + embedded_references, + ); } Ok(RbcInitialHeaderOutcome::StagedUnauthenticated { effects, error }) => { - let pinned = self.finish_header_staging(block_ref, Some(peer)); - self.notify_transaction_payload(peer, pinned, transaction_data); + // Preserve the content-addressed header for a later valid + // proof, but do not clear an exact recovery request or expose + // its payload as authoritative. Another advertised holder may + // still provide the receiver's valid MAC-vector entry. + match self.kernel.pinned_header(block_ref) { + Ok(Some(pinned)) => self.notify_header_staged(pinned), + Ok(None) => { + self.reject(Some(peer), RbcError::HeaderUnavailable(block_ref).into()) + } + Err(kernel_error) => self.reject(Some(peer), kernel_error.into()), + } self.process_effects(effects); self.reject(Some(peer), error.into()); } @@ -695,10 +774,16 @@ impl RbcServiceState { return; } match self.kernel.pinned_header(block_ref) { - Ok(Some(header)) => self.send_network( - peer, - NetworkMessage::RbcHeaderResponse(header.header().clone()), - ), + Ok(Some(header)) => { + if let Some(proposal) = self.retained_envelopes.get(&block_ref).cloned() { + self.send_network(peer, NetworkMessage::RbcHeaderEnvelopeResponse(proposal)); + } else { + self.send_network( + peer, + NetworkMessage::RbcHeaderResponse(header.header().clone()), + ); + } + } Ok(None) => {} Err(error) => self.reject(Some(peer), error.into()), } @@ -715,6 +800,16 @@ impl RbcServiceState { return; } let Some(fetch) = self.pending_fetches.get(&block_ref) else { + if self + .kernel + .pinned_header(block_ref) + .is_ok_and(|header| header.is_some()) + { + // A bounded fetch wave may have two holders in flight. Once + // the first exact response completes recovery, the second is + // an expected idempotent duplicate. + return; + } self.reject( Some(peer), RbcServiceError::UnexpectedHeaderResponse(block_ref), @@ -738,6 +833,36 @@ impl RbcServiceState { } } + fn accept_header_envelope_response( + &mut self, + peer: AuthorityIndex, + proposal: RbcHeaderProposal, + ) { + let block_ref = proposal.header().reference(); + let Some(fetch) = self.pending_fetches.get(&block_ref) else { + if self + .kernel + .pinned_header(block_ref) + .is_ok_and(|header| header.is_some()) + { + return; + } + self.reject( + Some(peer), + RbcServiceError::UnexpectedHeaderResponse(block_ref), + ); + return; + }; + if !fetch.holders.contains(peer) { + self.reject( + Some(peer), + RbcServiceError::HeaderResponseFromNonHolder { block_ref, peer }, + ); + return; + } + self.accept_direct_initial(peer, proposal); + } + fn finish_header_staging( &mut self, block_ref: BlockReference, @@ -776,6 +901,10 @@ impl RbcServiceState { } fn notify_header_staged(&mut self, header: PinnedRbcHeader) { + if sample_starfish_rbc_single_dag_phase(header.reference()) { + self.header_creation_times + .insert(header.reference(), header.header().meta_creation_time_ns()); + } if self.staged_notifications.insert(header.reference()) { let _ = self.events.send(RbcServiceEvent::HeaderStaged(header)); } @@ -796,9 +925,14 @@ impl RbcServiceState { RbcPhase::Echo => StarfishRbcReferenceKindV3::Echo, RbcPhase::Ready => StarfishRbcReferenceKindV3::Ready, }; - let _ = self.events.send(RbcServiceEvent::ReferenceReady( - StarfishRbcReferenceV3::new(kind, block_ref), - )); + let target_creation_time_ns = + sample_starfish_rbc_single_dag_phase(block_ref) + .then(|| self.header_creation_times.get(&block_ref).copied()) + .flatten(); + let _ = self.events.send(RbcServiceEvent::ReferenceReady { + reference: StarfishRbcReferenceV3::new(kind, block_ref), + target_creation_time_ns, + }); continue; } self.retained_phases.insert((block_ref, phase)); @@ -824,26 +958,9 @@ impl RbcServiceState { continue; } self.pending_fetches.remove(&header.reference()); + self.header_creation_times.remove(&header.reference()); let _ = self.events.send(RbcServiceEvent::Delivered(header)); } - RbcEffect::PortableEchoVote(target) => { - let RbcInitialAuthenticator::Mac(_, signer) = &self.initial_authenticator - else { - self.reject(None, RbcError::InvalidPortableEchoVote.into()); - continue; - }; - match self.kernel.portable_echo_signature_digest(target) { - Ok(digest) => { - let vote = StarfishRbcEchoVoteV3::new( - target, - self.own_authority, - signer.sign_digest(&digest), - ); - let _ = self.events.send(RbcServiceEvent::EchoVoteReady(vote)); - } - Err(error) => self.reject(None, error.into()), - } - } RbcEffect::PortableEchoQc(qc) => { let _ = self.events.send(RbcServiceEvent::EchoQcReady(qc)); } @@ -854,6 +971,7 @@ impl RbcServiceState { fn process_embedded_references( &mut self, sender: AuthorityIndex, + enclosing_block: BlockReference, references: Option, ) { if !matches!( @@ -870,19 +988,16 @@ impl RbcServiceState { return; }; for evidence in references.references() { - match self.kernel.handle_embedded_reference(sender, *evidence) { - Ok(effects) => self.process_effects(effects), - Err(error) => self.reject(Some(sender), error.into()), - } - } - for vote in references.echo_votes() { - match self.kernel.handle_portable_echo_vote(*vote) { + match self + .kernel + .handle_embedded_reference(sender, enclosing_block, *evidence) + { Ok(effects) => self.process_effects(effects), Err(error) => self.reject(Some(sender), error.into()), } } for qc in references.echo_qcs() { - match self.kernel.handle_portable_echo_qc(qc) { + match self.kernel.handle_portable_echo_qc(sender, qc) { Ok(effects) => self.process_effects(effects), Err(error) => self.reject(Some(sender), error.into()), } @@ -1022,8 +1137,7 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, - mac_keyrings_for_test, + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, starfish_rbc::RbcPhase, types::{TransactionData, VerifiedBlock}, @@ -1058,9 +1172,7 @@ mod tests { let keyrings = mac_keyrings_for_test(4); let authenticator = match scheme { BlockAuthenticationScheme::Ed25519 => RbcInitialAuthenticator::Ed25519(dummy_signer()), - BlockAuthenticationScheme::MacVector => { - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()) - } + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, BlockAuthenticationScheme::MlDsa44 => { RbcInitialAuthenticator::MlDsa44(dummy_ml_dsa_44_signer()) } @@ -1094,7 +1206,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedCarrierDag, @@ -1115,7 +1227,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedSingleDag { @@ -1138,7 +1250,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedSingleDag { @@ -1272,9 +1384,17 @@ mod tests { message: NetworkMessage::RbcInitial(_), .. } => initials += 1, - RbcServiceEvent::ReferenceReady(reference) => { + RbcServiceEvent::ReferenceReady { + reference, + target_creation_time_ns, + } => { assert_eq!(reference.kind(), StarfishRbcReferenceKindV3::Echo); assert_eq!(reference.reference(), canonical.reference()); + assert_eq!( + target_creation_time_ns, + sample_starfish_rbc_single_dag_phase(canonical.reference()) + .then_some(canonical.meta_creation_time_ns()) + ); references += 1; } RbcServiceEvent::Network { @@ -1292,27 +1412,32 @@ mod tests { } #[tokio::test] - async fn portable_fast_path_emits_signed_echo_vote_without_phase_message() { + async fn portable_fast_path_emits_signature_free_echo_witness_without_phase_message() { let (handle, mut events, task) = start_single_dag_fast_service(); let mut header = local_header(1, 4); header.starfish_rbc_v3 = Some(StarfishRbcFieldsV3::default()); let canonical = handle.start_local_header(header).await.unwrap(); let mut initials = 0; - let mut vote = None; + let mut echo_reference = None; for _ in 0..5 { match next_event(&mut events).await { RbcServiceEvent::HeaderStaged(header) => { assert_eq!(header.reference(), canonical.reference()); } RbcServiceEvent::Network { - message: NetworkMessage::RbcInitial(_), + message: NetworkMessage::RbcInitial(proposal), .. - } => initials += 1, - RbcServiceEvent::EchoVoteReady(echo_vote) => vote = Some(echo_vote), - RbcServiceEvent::ReferenceReady(reference) + } => { + let RbcInitialProof::MacVector(tags) = proposal.proof() else { + panic!("portable MAC mode must retain the complete MAC vector") + }; + assert_eq!(tags.len(), 4); + initials += 1; + } + RbcServiceEvent::ReferenceReady { reference, .. } if reference.kind() == StarfishRbcReferenceKindV3::Echo => { - panic!("portable mode emitted an unsigned ECHO reference") + echo_reference = Some(reference); } RbcServiceEvent::Network { message: NetworkMessage::RbcPhase(_), @@ -1322,9 +1447,10 @@ mod tests { } } assert_eq!(initials, 3); - let vote = vote.expect("portable mode must emit one signed ECHO vote"); - assert_eq!(vote.target(), canonical.reference()); - assert_eq!(vote.sender(), 0); + assert_eq!( + echo_reference.expect("portable mode must emit one embedded ECHO witness"), + StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, canonical.reference(),) + ); drop(handle); task.await.unwrap(); } @@ -1501,7 +1627,7 @@ mod tests { }; assert_eq!(proposal.header(), &canonical); let RbcInitialProof::Mac(tag) = proposal.proof() else { - panic!("MAC mode must send one tag") + panic!("strict MAC mode must send one recipient tag") }; initial_proofs.push(*tag); } @@ -1726,7 +1852,7 @@ mod tests { instance(), BlockAuthenticationScheme::Ed25519, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(1), ); diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index d5fd3f90..004fb5b6 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -30,7 +30,7 @@ use crate::{ types::{ AuthorityIndex, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishNoVoteCert, SailfishTimeoutCert, Stake, - VerifiedBlock, + TimestampNs, VerifiedBlock, }, }; @@ -405,13 +405,10 @@ impl Syncer { pub fn apply_starfish_rbc_reference( &mut self, reference: crate::types::StarfishRbcReferenceV3, + target_creation_time_ns: Option, ) { - self.core.add_starfish_rbc_reference(reference); - self.try_new_block(BlockCreationReason::CertificateEvent); - } - - pub fn apply_starfish_rbc_echo_vote(&mut self, vote: crate::types::StarfishRbcEchoVoteV3) { - self.core.add_starfish_rbc_echo_vote(vote); + self.core + .add_starfish_rbc_reference(reference, target_creation_time_ns); self.try_new_block(BlockCreationReason::CertificateEvent); } diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 2afdb77a..082e0af5 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -138,75 +138,29 @@ pub struct StarfishRbcReferenceV3 { reference: BlockReference, } -/// A publicly verifiable ECHO vote carried by an ordinary single-DAG block. -/// -/// The signature is deliberately independent of the carrying block. A later -/// block can therefore copy a quorum of votes into a portable certificate -/// without introducing a standalone RBC phase message. -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] -pub struct StarfishRbcEchoVoteV3 { - target: BlockReference, - sender: AuthorityIndex, - signature: BlsSignatureBytes, -} - -impl StarfishRbcEchoVoteV3 { - pub fn new( - target: BlockReference, - sender: AuthorityIndex, - signature: BlsSignatureBytes, - ) -> Self { - Self { - target, - sender, - signature, - } - } - - pub fn target(self) -> BlockReference { - self.target - } - - pub fn sender(self) -> AuthorityIndex { - self.sender - } - - pub fn signature(self) -> BlsSignatureBytes { - self.signature - } -} - -/// Portable quorum certificate over exact, publicly verifiable ECHO votes. +/// Signature-free portable quorum certificate over exact ordinary-DAG blocks +/// that carry `ECHO(target)`. Every witness block is independently +/// authenticated with its author's complete MAC vector; a holder can relay +/// that vector and each receiver verifies only its own entry. #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct StarfishRbcEchoQcV3 { target: BlockReference, - signers: AuthoritySet, - signature: BlsSignatureBytes, + witnesses: Vec, } impl StarfishRbcEchoQcV3 { - pub fn new( - target: BlockReference, - signers: AuthoritySet, - signature: BlsSignatureBytes, - ) -> Self { - Self { - target, - signers, - signature, - } + pub fn new(target: BlockReference, mut witnesses: Vec) -> Self { + witnesses.sort_unstable(); + witnesses.dedup(); + Self { target, witnesses } } pub fn target(&self) -> BlockReference { self.target } - pub fn signers(&self) -> AuthoritySet { - self.signers - } - - pub fn signature(&self) -> BlsSignatureBytes { - self.signature + pub fn witnesses(&self) -> &[BlockReference] { + &self.witnesses } } @@ -233,8 +187,6 @@ impl StarfishRbcReferenceV3 { pub struct StarfishRbcFieldsV3 { references: Vec, #[serde(default)] - echo_votes: Vec, - #[serde(default)] echo_qcs: Vec, } @@ -244,25 +196,20 @@ impl StarfishRbcFieldsV3 { references.dedup(); Self { references, - echo_votes: Vec::new(), echo_qcs: Vec::new(), } } pub fn with_portable_echo( mut references: Vec, - mut echo_votes: Vec, mut echo_qcs: Vec, ) -> Self { references.sort_unstable(); references.dedup(); - echo_votes.sort_unstable(); - echo_votes.dedup(); echo_qcs.sort_unstable(); echo_qcs.dedup(); Self { references, - echo_votes, echo_qcs, } } @@ -271,10 +218,6 @@ impl StarfishRbcFieldsV3 { &self.references } - pub fn echo_votes(&self) -> &[StarfishRbcEchoVoteV3] { - &self.echo_votes - } - pub fn echo_qcs(&self) -> &[StarfishRbcEchoQcV3] { &self.echo_qcs } @@ -287,9 +230,7 @@ impl StarfishRbcFieldsV3 { if self.references.len() > committee.len().saturating_mul(6) { return false; } - if self.echo_votes.len() > committee.len().saturating_mul(3) - || self.echo_qcs.len() > committee.len() - { + if self.echo_qcs.len() > committee.len() { return false; } if self.references.windows(2).any(|pair| pair[0] >= pair[1]) { @@ -303,38 +244,34 @@ impl StarfishRbcFieldsV3 { && committee.known_authority(reference.authority) && statements.insert((evidence.kind(), reference.authority, reference.round)) }); - if !references_valid - || self.echo_votes.windows(2).any(|pair| pair[0] >= pair[1]) - || self.echo_qcs.windows(2).any(|pair| pair[0] >= pair[1]) - { + if !references_valid || self.echo_qcs.windows(2).any(|pair| pair[0] >= pair[1]) { return false; } - let votes_valid = self.echo_votes.iter().all(|vote| { - let target = vote.target(); - target.round > 0 - && target.round <= block_round - && committee.known_authority(target.authority) - && committee.known_authority(vote.sender()) - }); - votes_valid - && self.echo_qcs.iter().all(|qc| { - let target = qc.target(); - if target.round == 0 - || target.round >= block_round - || !committee.known_authority(target.authority) - || qc.signers().is_empty() + self.echo_qcs.iter().all(|qc| { + let target = qc.target(); + if target.round == 0 + || target.round >= block_round + || !committee.known_authority(target.authority) + || qc.witnesses().is_empty() + || qc.witnesses().len() > committee.len() + || qc.witnesses().windows(2).any(|pair| pair[0] >= pair[1]) + { + return false; + } + let mut stake = 0; + let mut authors = AHashSet::new(); + for witness in qc.witnesses() { + if witness.round == 0 + || witness.round >= block_round + || !committee.known_authority(witness.authority) + || !authors.insert(witness.authority) { return false; } - let mut stake = 0; - for sender in qc.signers().present() { - if !committee.known_authority(sender) { - return false; - } - stake += committee.get_stake(sender).unwrap_or_default(); - } - committee.is_quorum(stake) - }) + stake += committee.get_stake(witness.authority).unwrap_or_default(); + } + committee.is_quorum(stake) + }) } } diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 8a8f14ed..ad65b8af 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -210,8 +210,8 @@ enum Operation { /// Requires embedded RBC-DAG authority and changes the finality proof. #[clap(long, default_value_t = false)] starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only: deliver a single-DAG RBC header from a portable - /// quorum of publicly signed ECHO votes carried in ordinary blocks. + /// Testbed-only: deliver a single-DAG RBC header from exact ordinary + /// DAG blocks carrying ECHO, authenticated only by MAC vectors. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. @@ -615,7 +615,7 @@ async fn local_benchmark( } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( - "Single-DAG portable ECHO-QC delivery: ENABLED (signed votes; testbed evaluation)" + "Single-DAG MAC-witness ECHO-QC: ENABLED (signature-free; testbed-only totality assumption)" ); } if let Some(latency) = node_parameters.uniform_latency_ms { diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index 233a637b..2b26eb16 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -37,8 +37,8 @@ is one of: The carrying block's authenticated author is the statement sender. The list is part of that ordinary block's content digest. Consequently the default path needs no standalone phase MAC, signature, or phase network message. The -portable fast path adds embedded BLS ECHO votes and an aggregate QC, but still -adds no phase message. +flagged MAC-witness experiment adds exact references to ECHO-carrying ordinary +blocks, but still adds neither a signature nor a phase message. References are sorted, duplicate-free and bounded by `6 * committee_size` per block. A sender may name at most one digest for each `(phase, target author, @@ -92,26 +92,26 @@ create a second physical round counter. ### Portable ECHO-QC fast path Finite benchmarks may opt into -`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode ECHO is a compact -BLS vote over the exact target reference, protocol instance and committee. -Votes ride in ordinary DAG blocks. Once quorum stake verifies, the node batch -aggregates them into one portable certificate containing the target, signer -bitmap and 48-byte aggregate signature. The QC also rides in an ordinary DAG -block; there is no standalone ECHO or READY message. - -An honest node publishes or relays the QC before its delivery effect. The -ordered Core bridge queues the QC-bearing block before applying delivery. -Quorum intersection gives uniqueness, while public verification and mandatory -relay remove the old receiver-local selective-withholding caveat: any valid QC -that reaches one honest validator can be verified and propagated by every -other validator. Missing target content still uses exact header recovery and -delivery remains fail-closed until that content is present. Invalid aggregate -batches fall back to individual vote verification so one Byzantine vote cannot -poison an otherwise valid quorum. - -The flag remains testbed-only because bounded retirement, durable pending-QC -replay and a complete asynchronous proof are still production follow-ups. The -default V3 path continues to use the conventional quorum-READY rule. +`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode a certificate is the +target plus a quorum of exact ordinary-block references whose authenticated +headers contain `Echo(target)`. It contains no BLS, Ed25519, ML-DSA or other +digital signature. The certificate rides in an ordinary DAG block; there is no +standalone ECHO or READY message. + +For synchronization, fast-path MAC proposals retain the author's complete MAC +vector. A holder may relay the exact header and vector, and each receiver checks +only its own pairwise entry before accepting that witness. Normal delivery adds +no message; exact header/vector requests occur only when a named witness is +missing. + +This flag is deliberately a **latency lower bound, not the Byzantine-totality +mode**. A Byzantine witness author can make its vector valid for the first QC +builder but invalid for another receiver. Requiring every named quorum witness +therefore can prevent that receiver from accepting an otherwise observed QC; +accepting fewer witnesses would instead lose quorum-intersection safety. Public +signatures solve transferability but violate the signature-free requirement. +The default quorum-ECHO then quorum-READY path remains the signature-free, +totality-safe candidate. ## Required validation @@ -133,11 +133,16 @@ with MAC authentication and the fixed 50 ms V3 round limiter: | `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | | V3 + old receiver-local ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | | V3 + old receiver-local ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | -| V3 + portable aggregate ECHO-QC | zero | 405.8 ms | 1,000 | 1.06 MB/s | -| V3 + portable aggregate ECHO-QC | AWS table | 983.7 ms | 1,000 | 0.74 MB/s | +| V3 + rejected BLS aggregate ECHO-QC | zero | 405.8 ms | 1,000 | 1.06 MB/s | +| V3 + rejected BLS aggregate ECHO-QC | AWS table | 983.7 ms | 1,000 | 0.74 MB/s | +| V3 + signature-free MAC-witness ECHO-QC | zero | 397.1 ms | 1,000 | 1.38 MB/s | +| V3 + signature-free MAC-witness ECHO-QC | AWS table | 955.3 ms | 1,000 | 0.91 MB/s | | `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | All exact offered transactions committed during the bounded drain. These are single-machine research measurements, not production claims. The two old -receiver-local rows preserve the historical totality caveat; the new portable -rows use the signed aggregate certificate described above. +receiver-local rows preserve the historical totality caveat. The BLS rows are +retained only as historical latency data and are not part of the signature-free +design. The MAC-witness rows are also a flagged lower bound: unlike the strict +default path, they assume every named witness supplied a valid complete MAC +vector for every honest receiver. From 4f97f8ef56c353fb147214b82fb8a72537e27e0a Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:19:28 +0200 Subject: [PATCH 48/62] Revert "replace portable echo signatures with MAC witnesses" This reverts commit 3ad9bffe0841c9de005e7da1a7036f11e10997fc. --- crates/starfish-core/src/config.rs | 7 +- crates/starfish-core/src/core.rs | 104 +-- .../starfish-core/src/core_thread/spawned.rs | 39 +- crates/starfish-core/src/crypto.rs | 37 +- crates/starfish-core/src/metrics.rs | 142 +--- crates/starfish-core/src/net_sync.rs | 43 +- crates/starfish-core/src/network.rs | 5 - crates/starfish-core/src/starfish_rbc.rs | 669 ++++++++---------- .../starfish-core/src/starfish_rbc_service.rs | 306 +++----- crates/starfish-core/src/syncer.rs | 11 +- crates/starfish-core/src/types.rs | 133 +++- crates/starfish/src/main.rs | 6 +- docs/starfish-rbc-single-dag-v3.md | 57 +- 13 files changed, 605 insertions(+), 954 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index c99cd01a..7701db4d 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -94,10 +94,9 @@ pub struct NodeParameters { /// Starfish finality proof shape. #[serde(default)] pub starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed single-DAG RBC path: exact ordinary DAG blocks carrying ECHO - /// form a signature-free MAC-witness certificate. This is a latency lower - /// bound: a Byzantine witness can selectively corrupt another receiver's - /// MAC-vector entry, so strict ECHO->READY remains the totality-safe mode. + /// Testbed single-DAG RBC path: publicly signed ECHO votes are copied into + /// a portable quorum certificate carried by an ordinary DAG block. This + /// preserves uniqueness and totality without a standalone phase message. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index cba8bf3c..fe8b6fc9 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -2,11 +2,7 @@ // Modifications Copyright (c) 2025 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt, mem, - sync::Arc, -}; +use std::{collections::BTreeSet, fmt, mem, sync::Arc}; use ahash::{AHashMap, AHashSet}; use reed_solomon_simd::ReedSolomonEncoder; @@ -43,8 +39,8 @@ use crate::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishFields, Shard, - StarfishRbcEchoQcV3, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, - StarfishRbcReferenceV3, TimestampNs, VerifiedBlock, + StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, StarfishRbcReferenceV3, + VerifiedBlock, }, }; @@ -62,12 +58,7 @@ pub struct Core { /// Irrevocable local ECHO/READY statements waiting to ride on the next /// ordinary block in the single-DAG protocol. pending_starfish_rbc_references: BTreeSet, - /// Local enqueue timestamp for each pending signature-free ECHO/READY - /// statement. This is diagnostic-only and lets the benchmark distinguish - /// network quorum time from time spent waiting for the next ordinary DAG - /// block without adding protocol messages or changing block contents. - pending_starfish_rbc_reference_queued_at_ns: - BTreeMap, + pending_starfish_rbc_echo_votes: BTreeSet, pending_starfish_rbc_echo_qcs: BTreeSet, // For Byzantine node, last_own_block contains a vector of blocks last_own_block: Vec, @@ -335,7 +326,7 @@ impl Core { pending, pending_reconstructed_data: AHashMap::new(), pending_starfish_rbc_references: BTreeSet::new(), - pending_starfish_rbc_reference_queued_at_ns: BTreeMap::new(), + pending_starfish_rbc_echo_votes: BTreeSet::new(), pending_starfish_rbc_echo_qcs: BTreeSet::new(), last_own_block: vec![last_own_block], block_handler, @@ -376,35 +367,23 @@ impl Core { &self.bls_signer } - pub(crate) fn add_starfish_rbc_reference( - &mut self, - reference: StarfishRbcReferenceV3, - target_creation_time_ns: Option, - ) { + pub(crate) fn add_starfish_rbc_reference(&mut self, reference: StarfishRbcReferenceV3) { assert!( self.dag_state .consensus_protocol .is_starfish_rbc_single_dag(), "embedded RBC references require single-DAG Starfish-RBC" ); - if self.pending_starfish_rbc_references.insert(reference) { - if let Some(target_creation_time_ns) = target_creation_time_ns { - let now_ns = timestamp_utc() - .as_nanos() - .try_into() - .unwrap_or(TimestampNs::MAX); - self.pending_starfish_rbc_reference_queued_at_ns - .insert(reference, (now_ns, target_creation_time_ns)); - self.metrics - .observe_starfish_rbc_single_dag_phase_target_age_ns( - match reference.kind() { - StarfishRbcReferenceKindV3::Echo => "creation_to_echo_queued", - StarfishRbcReferenceKindV3::Ready => "creation_to_ready_queued", - }, - Some(now_ns.saturating_sub(target_creation_time_ns)), - ); - } - } + self.pending_starfish_rbc_references.insert(reference); + } + + pub(crate) fn add_starfish_rbc_echo_vote(&mut self, vote: StarfishRbcEchoVoteV3) { + assert!( + self.dag_state + .consensus_protocol + .is_starfish_rbc_single_dag() + ); + self.pending_starfish_rbc_echo_votes.insert(vote); } pub(crate) fn add_starfish_rbc_echo_qc(&mut self, qc: StarfishRbcEchoQcV3) { @@ -895,9 +874,6 @@ impl Core { }; let single_dag_rbc = protocol.is_starfish_rbc_single_dag().then(|| { let maximum = self.committee.len().saturating_mul(6); - self.metrics.set_starfish_rbc_single_dag_pending_references( - self.pending_starfish_rbc_references.len(), - ); let references: Vec<_> = self .pending_starfish_rbc_references .iter() @@ -907,52 +883,28 @@ impl Core { .collect(); for reference in &references { self.pending_starfish_rbc_references.remove(reference); - let queued_timing = self - .pending_starfish_rbc_reference_queued_at_ns - .remove(reference); - if let Some((queued_at_ns, target_creation_ns)) = queued_timing { - let now_ns = timestamp_utc() - .as_nanos() - .try_into() - .unwrap_or(TimestampNs::MAX); - self.metrics - .observe_starfish_rbc_single_dag_phase_target_age_ns( - match reference.kind() { - StarfishRbcReferenceKindV3::Echo => "creation_to_echo_embedded", - StarfishRbcReferenceKindV3::Ready => "creation_to_ready_embedded", - }, - Some(now_ns.saturating_sub(target_creation_ns)), - ); - self.metrics - .observe_starfish_rbc_single_dag_phase_target_age_ns( - match reference.kind() { - StarfishRbcReferenceKindV3::Echo => "echo_queue_dwell", - StarfishRbcReferenceKindV3::Ready => "ready_queue_dwell", - }, - Some(now_ns.saturating_sub(queued_at_ns)), - ); - } } - self.metrics.set_starfish_rbc_single_dag_pending_references( - self.pending_starfish_rbc_references.len(), - ); + let echo_votes: Vec<_> = self + .pending_starfish_rbc_echo_votes + .iter() + .filter(|vote| vote.target().round <= clock_round) + .take(self.committee.len().saturating_mul(3)) + .copied() + .collect(); + for vote in &echo_votes { + self.pending_starfish_rbc_echo_votes.remove(vote); + } let echo_qcs: Vec<_> = self .pending_starfish_rbc_echo_qcs .iter() - .filter(|qc| { - qc.target().round < clock_round - && qc - .witnesses() - .iter() - .all(|witness| witness.round < clock_round) - }) + .filter(|qc| qc.target().round < clock_round) .take(self.committee.len()) .cloned() .collect(); for qc in &echo_qcs { self.pending_starfish_rbc_echo_qcs.remove(qc); } - StarfishRbcFieldsV3::with_portable_echo(references, echo_qcs) + StarfishRbcFieldsV3::with_portable_echo(references, echo_votes, echo_qcs) }); // Create and store blocks diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 286773fa..86728659 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -81,11 +81,8 @@ enum CoreThreadCommand { ApplySailfishCertificates(Vec, oneshot::Sender<()>), /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), - ApplyStarfishRbcReference( - crate::types::StarfishRbcReferenceV3, - Option, - oneshot::Sender<()>, - ), + ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), + ApplyStarfishRbcEchoVote(crate::types::StarfishRbcEchoVoteV3, oneshot::Sender<()>), ApplyStarfishRbcEchoQc(crate::types::StarfishRbcEchoQcV3, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. ApplyStarfishRbcDagFrontier( @@ -281,18 +278,25 @@ impl, ) { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ApplyStarfishRbcReference( - reference, - target_creation_time_ns, - sender, + reference, sender, )) .await; receiver.await.expect("core thread is not expected to stop"); } + pub(crate) async fn apply_starfish_rbc_echo_vote( + &self, + vote: crate::types::StarfishRbcEchoVoteV3, + ) { + let (sender, receiver) = oneshot::channel(); + self.send(CoreThreadCommand::ApplyStarfishRbcEchoVote(vote, sender)) + .await; + receiver.await.expect("core thread is not expected to stop"); + } + pub(crate) async fn apply_starfish_rbc_echo_qc(&self, qc: crate::types::StarfishRbcEchoQcV3) { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ApplyStarfishRbcEchoQc(qc, sender)) @@ -538,17 +542,20 @@ impl CoreThread { self.syncer.apply_starfish_rbc_deliveries(delivered_headers); sender.send(()).ok(); } - CoreThreadCommand::ApplyStarfishRbcReference( - reference, - target_creation_time_ns, - sender, - ) => { + CoreThreadCommand::ApplyStarfishRbcReference(reference, sender) => { metrics .core_thread_tasks_total .with_label_values(&["apply_starfish_rbc_reference"]) .inc(); - self.syncer - .apply_starfish_rbc_reference(reference, target_creation_time_ns); + self.syncer.apply_starfish_rbc_reference(reference); + sender.send(()).ok(); + } + CoreThreadCommand::ApplyStarfishRbcEchoVote(vote, sender) => { + metrics + .core_thread_tasks_total + .with_label_values(&["apply_starfish_rbc_echo_vote"]) + .inc(); + self.syncer.apply_starfish_rbc_echo_vote(vote); sender.send(()).ok(); } CoreThreadCommand::ApplyStarfishRbcEchoQc(qc, sender) => { diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index cf013da6..27e38796 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -326,19 +326,25 @@ impl BlockDigest { // Preserve the frozen V3 digest exactly when the portable extension // is absent. A non-empty extension is explicitly tagged before its // length-delimited fields, so old stored/default blocks still reopen. - if !rbc.echo_qcs().is_empty() { - hasher.update(b"PORTABLE_ECHO_WITNESS_QC_V1"); + if !rbc.echo_votes().is_empty() || !rbc.echo_qcs().is_empty() { + hasher.update(b"PORTABLE_ECHO_QC_V1"); + let vote_len = u32::try_from(rbc.echo_votes().len()) + .expect("Starfish-RBC ECHO vote count exceeds u32"); + hasher.update(&vote_len.to_be_bytes()); + for vote in rbc.echo_votes() { + hash_reference(&mut hasher, &vote.target()); + hasher.update(&vote.sender().to_be_bytes()); + hasher.update(vote.signature().as_ref()); + } let qc_len = u32::try_from(rbc.echo_qcs().len()) .expect("Starfish-RBC ECHO-QC count exceeds u32"); hasher.update(&qc_len.to_be_bytes()); for qc in rbc.echo_qcs() { hash_reference(&mut hasher, &qc.target()); - let witness_len = u32::try_from(qc.witnesses().len()) - .expect("Starfish-RBC ECHO-QC witness count exceeds u32"); - hasher.update(&witness_len.to_be_bytes()); - for witness in qc.witnesses() { - hash_reference(&mut hasher, witness); + for word in qc.signers().words() { + hasher.update(&word.to_be_bytes()); } + hasher.update(qc.signature().as_ref()); } } Self(hasher.finalize().into()) @@ -1381,6 +1387,23 @@ pub fn bls_aggregate(sigs: &[&BlsSignatureBytes]) -> BlsSignatureBytes { BlsSignatureBytes(agg.to_signature().to_bytes()) } +/// Fallible counterpart used on untrusted wire votes before they have been +/// individually verified. The returned aggregate is still untrusted and must +/// be verified against the exact signer set and message. +pub fn bls_try_aggregate(sigs: &[&BlsSignatureBytes]) -> Option { + if sigs.is_empty() { + return None; + } + let parsed: Vec = sigs + .iter() + .map(|signature| bls::Signature::from_bytes(&signature.0)) + .collect::>() + .ok()?; + let references: Vec<_> = parsed.iter().collect(); + let aggregate = bls::AggregateSignature::aggregate(&references, true).ok()?; + Some(BlsSignatureBytes(aggregate.to_signature().to_bytes())) +} + /// Verify an aggregate signature against multiple public keys (all signed same /// message). #[allow(dead_code)] diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 7d97702c..564fdfd9 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -30,7 +30,7 @@ use crate::{ EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, }, stat::{DivUsize, HistogramSender, PreciseHistogram, histogram}, - types::{AuthorityIndex, BlockReference, format_authority_index}, + types::{AuthorityIndex, format_authority_index}, }; /// Metrics collected by the benchmark. @@ -150,42 +150,6 @@ const RBC_DAG_PIPELINE_LATENCY_STAGES: &[&str] = &[ RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, ]; -const RBC_SINGLE_DAG_PHASE_LATENCY_STAGES: &[&str] = &[ - "creation_to_echo_queued", - "creation_to_echo_embedded", - "echo_queue_dwell", - "creation_to_ready_queued", - "creation_to_ready_embedded", - "ready_queue_dwell", - "creation_to_delivery", -]; - -#[derive(Default)] -struct SingleDagPhaseTiming { - total_ns: [AtomicU64; 7], - samples: [AtomicU64; 7], - max_ns: [AtomicU64; 7], -} - -fn single_dag_phase_index(stage: &'static str) -> usize { - match stage { - "creation_to_echo_queued" => 0, - "creation_to_echo_embedded" => 1, - "echo_queue_dwell" => 2, - "creation_to_ready_queued" => 3, - "creation_to_ready_embedded" => 4, - "ready_queue_dwell" => 5, - "creation_to_delivery" => 6, - _ => panic!("unknown single-DAG phase timing stage {stage}"), - } -} - -/// Deterministic 1/16 diagnostic sampling by the already-random block digest. -/// The protocol path and every threshold still process all references; only -/// timing observation is sampled to keep a 40-validator local run measurable. -pub(crate) fn sample_starfish_rbc_single_dag_phase(reference: BlockReference) -> bool { - reference.digest.as_ref()[0] & 0x0f == 0 -} pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD: &str = "physical_forward"; pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD: &str = "physical_backward"; const RBC_DAG_COMMIT_DISTANCE_KINDS: &[&str] = &[ @@ -302,12 +266,6 @@ pub struct Metrics { pub network_message_bytes_sent_total: IntCounterVec, pub network_message_bytes_received_total: IntCounterVec, - /// Signature-free single-DAG RBC phase timing. Fixed lock-free slots keep - /// this diagnostic off the Prometheus label/map hot path at n=40. - starfish_rbc_single_dag_phase_timing: Arc, - starfish_rbc_single_dag_pending_references: Arc, - starfish_rbc_single_dag_pending_references_max: Arc, - // Starfish-RBC-DAG shadow instrumentation. These metrics are strictly // observational: the shadow path never feeds the authoritative DAG or // consensus state. @@ -736,33 +694,6 @@ fn format_rbc_dag_round_distance(total: u64, samples: u64, maximum: i64) -> Stri } impl Metrics { - pub(crate) fn observe_starfish_rbc_single_dag_phase_target_age_ns( - &self, - stage: &'static str, - latency_ns: Option, - ) { - let Some(latency_ns) = latency_ns else { - return; - }; - if !self.transaction_metrics_active.load(Ordering::Relaxed) { - return; - } - let index = single_dag_phase_index(stage); - self.starfish_rbc_single_dag_phase_timing.total_ns[index] - .fetch_add(latency_ns, Ordering::Relaxed); - self.starfish_rbc_single_dag_phase_timing.samples[index].fetch_add(1, Ordering::Relaxed); - self.starfish_rbc_single_dag_phase_timing.max_ns[index] - .fetch_max(latency_ns, Ordering::Relaxed); - } - - pub(crate) fn set_starfish_rbc_single_dag_pending_references(&self, pending: usize) { - let pending = u64::try_from(pending).unwrap_or(u64::MAX); - self.starfish_rbc_single_dag_pending_references - .store(pending, Ordering::Relaxed); - self.starfish_rbc_single_dag_pending_references_max - .fetch_max(pending, Ordering::Relaxed); - } - pub(crate) fn observe_starfish_rbc_dag_pipeline_latency_ns( &self, stage: &'static str, @@ -1298,9 +1229,6 @@ impl Metrics { registry, ) .unwrap(), - starfish_rbc_single_dag_phase_timing: Arc::new(SingleDagPhaseTiming::default()), - starfish_rbc_single_dag_pending_references: Arc::new(AtomicU64::new(0)), - starfish_rbc_single_dag_pending_references_max: Arc::new(AtomicU64::new(0)), starfish_rbc_dag_shadow_inputs_total: register_int_counter_vec_with_registry!( "starfish_rbc_dag_shadow_inputs_total", "Starfish-RBC-DAG shadow inputs, by bounded input kind and processing outcome", @@ -2179,52 +2107,6 @@ impl Metrics { .sum::() .as_millis() as f64 / num_validators as f64; - let single_dag_phase_latencies = RBC_SINGLE_DAG_PHASE_LATENCY_STAGES - .iter() - .enumerate() - .filter_map(|(index, stage)| { - let total_ns = metrics - .iter() - .map(|metrics| { - metrics.starfish_rbc_single_dag_phase_timing.total_ns[index] - .load(Ordering::Relaxed) - }) - .sum::(); - let samples = metrics - .iter() - .map(|metrics| { - metrics.starfish_rbc_single_dag_phase_timing.samples[index] - .load(Ordering::Relaxed) - }) - .sum::(); - if samples == 0 { - return None; - } - let maximum_ns = metrics - .iter() - .map(|metrics| { - metrics.starfish_rbc_single_dag_phase_timing.max_ns[index] - .load(Ordering::Relaxed) - }) - .max() - .unwrap_or_default(); - Some(( - *stage, - total_ns as f64 / samples as f64 / 1_000_000.0, - maximum_ns as f64 / 1_000_000.0, - samples, - )) - }) - .collect::>(); - let single_dag_pending_max = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_single_dag_pending_references_max - .load(Ordering::Relaxed) - }) - .max() - .unwrap_or_default(); let mut table = PrettyTable::new(); table.set_format(default_table_format()); @@ -2276,20 +2158,6 @@ impl Metrics { table.add_row(row![b->"Average TPS:", format!("{:.2} tx/s", average_tps)]); } table.add_row(row![b->"Average BPS:", format!("{:.2} blocks/s", average_bps)]); - if !single_dag_phase_latencies.is_empty() { - table.add_row(row![bH2->""]); - table.add_row(row![bH2->"Signature-Free Single-DAG RBC Phase Timing"]); - for (stage, average_ms, maximum_ms, samples) in single_dag_phase_latencies { - table.add_row(row![ - b->format!("{stage}:"), - format!("avg {average_ms:.2} ms, max {maximum_ms:.2} ms (n={samples})") - ]); - } - table.add_row(row![ - b->"Maximum pending ECHO/READY statements:", - single_dag_pending_max - ]); - } // Network metrics table.add_row(row![bH2->""]); @@ -3340,14 +3208,6 @@ mod tests { .with_label_values(&["direct_commit"]) .inc(); metrics.metrics_active.store(true, Ordering::Relaxed); - metrics - .transaction_metrics_active - .store(true, Ordering::Relaxed); - metrics.observe_starfish_rbc_single_dag_phase_target_age_ns( - "creation_to_ready_queued", - Some(11), - ); - metrics.set_starfish_rbc_single_dag_pending_references(2); metrics.observe_starfish_rbc_dag_pipeline_latency_ns( RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, 12, diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 2650651b..1c95a86f 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -43,7 +43,7 @@ use crate::{ metrics::{ Metrics, RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, - UtilizationTimerVecExt, sample_starfish_rbc_single_dag_phase, + UtilizationTimerVecExt, }, network::{BlockBatch, Connection, Network, NetworkMessage, RbcDagShadowCarrier, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, @@ -1732,15 +1732,6 @@ impl ConnectionHandler { - if let Some(ref rbc) = self.starfish_rbc_service { - if let Err(error) = rbc.header_envelope_response(self.peer_id, proposal) { - tracing::warn!( - "Failed to forward Starfish-RBC authenticated header response: {error}" - ); - } - } - } NetworkMessage::RbcDagShadowCarrier(envelope) => { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier_reliably(self.peer_id, envelope).await { @@ -3201,7 +3192,10 @@ impl NetworkSyncer BlockAuthenticationScheme::MlDsa65 => { RbcInitialAuthenticator::MlDsa65(core.get_ml_dsa_65_signer().clone()) } - BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac( + core.get_signer().clone(), + core.get_bls_signer().clone(), + ), }; let (service, events, task) = start_starfish_rbc_service_with_phase_authority( committee.clone(), @@ -3548,17 +3542,6 @@ impl NetworkSyncer .await; } RbcServiceEvent::Delivered(header) => { - if sample_starfish_rbc_single_dag_phase(header.reference()) { - let now_ns = current_timestamp_ns(); - rbc_metrics.observe_starfish_rbc_single_dag_phase_target_age_ns( - "creation_to_delivery", - Some( - now_ns.saturating_sub( - header.header().meta_creation_time_ns(), - ), - ), - ); - } if let Some(ref shadow) = event_inner.starfish_rbc_dag_shadow_service { @@ -3582,16 +3565,16 @@ impl NetworkSyncer .apply_starfish_rbc_deliveries(vec![header]) .await; } - RbcServiceEvent::ReferenceReady { - reference, - target_creation_time_ns, - } => { + RbcServiceEvent::ReferenceReady(reference) => { event_inner .syncer - .apply_starfish_rbc_reference( - reference, - target_creation_time_ns, - ) + .apply_starfish_rbc_reference(reference) + .await; + } + RbcServiceEvent::EchoVoteReady(vote) => { + event_inner + .syncer + .apply_starfish_rbc_echo_vote(vote) .await; } RbcServiceEvent::EchoQcReady(qc) => { diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 1913fb55..63799242 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -295,10 +295,6 @@ pub enum NetworkMessage { /// authentication-sidecar variant. Appended after the frozen V1 message /// family so every preceding bincode enum discriminant remains stable. RbcDagShadowCarrierEnvelopeResponse(RbcDagShadowCarrierEnvelopeResponse), - /// Signature-free single-DAG recovery response carrying the exact header, - /// optional payload, and the author's complete MAC vector. Appended so - /// every preceding bincode discriminant remains stable. - RbcHeaderEnvelopeResponse(RbcHeaderProposal), } impl NetworkMessage { @@ -336,7 +332,6 @@ impl NetworkMessage { Self::RbcDagShadowCarrierEnvelopeResponse(_) => { "rbc_dag_shadow_carrier_envelope_response" } - Self::RbcHeaderEnvelopeResponse(_) => "rbc_header_envelope_response", } } } diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index a14f66d0..feeb9446 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -7,12 +7,7 @@ //! supplies content-validated headers and expands typed multicast effects into //! recipient-specific messages. -use std::{ - collections::{BTreeMap, BTreeSet}, - error::Error, - fmt, - sync::Arc, -}; +use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; use ahash::{AHashMap, AHashSet}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -21,14 +16,15 @@ use crate::{ committee::{Committee, QuorumThreshold, StakeAggregator, ValidityThreshold}, crypto::{ Blake3Hasher, MacKey, MacTag, MlDsa44SignatureBytes, MlDsa65SignatureBytes, SignatureBytes, - TransactionsCommitment, + TransactionsCommitment, bls_aggregate, bls_fast_aggregate_verify, + bls_public_keys_for_signers, bls_try_aggregate, }, types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - StarfishRbcEchoQcV3, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, - StarfishRbcReferenceV3, TimestampNs, TransactionData, VerifiedBlock, - compress_acknowledgments, expand_acknowledgments, + StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, + StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, + VerifiedBlock, compress_acknowledgments, expand_acknowledgments, }, }; @@ -37,6 +33,7 @@ const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_V1_COMMITTEE_ID"; const INITIAL_KIND: u8 = 0x00; const ECHO_KIND: u8 = 0x01; const READY_KIND: u8 = 0x02; +const PORTABLE_ECHO_KIND: u8 = 0x03; const PROTOCOL_INSTANCE_SIZE: usize = 32; const COMMITTEE_ID_SIZE: usize = 32; @@ -530,16 +527,21 @@ impl RbcCanonicalHeader { }) .ok_or(RbcError::HeaderContentTooLarge)?; let portable_echo_bytes = self.starfish_rbc_v3.as_ref().map_or(0, |rbc| { + let vote_bytes: usize = rbc + .echo_votes() + .iter() + .map(|vote| RBC_BLOCK_REFERENCE_SIZE + 2 + vote.signature().as_ref().len()) + .sum(); let qc_bytes: usize = rbc .echo_qcs() .iter() .map(|qc| { RBC_BLOCK_REFERENCE_SIZE - + std::mem::size_of::() - + qc.witnesses().len() * RBC_BLOCK_REFERENCE_SIZE + + qc.signers().words().len() * std::mem::size_of::() + + qc.signature().as_ref().len() }) .sum(); - qc_bytes + vote_bytes.saturating_add(qc_bytes) }); RBC_BLOCK_REFERENCE_SIZE .checked_mul(reference_count) @@ -599,14 +601,9 @@ pub enum RbcInitialProof { MlDsa44(MlDsa44SignatureBytes), MlDsa65(MlDsa65SignatureBytes), Mac(MacTag), - /// Complete author-generated MAC vector. Unlike a single recipient tag, - /// this proof may be relayed: each receiver verifies only its own entry. - MacVector(Vec), } -/// Starfish-RBC header proposal carried on the wire. Recipient-tag proofs are -/// direct-author-only; a complete MAC-vector proof may be relayed during exact -/// witness recovery. +/// Direct-author Starfish-RBC header proposal carried on the wire. /// /// The proof is a sidecar over the canonical header reference. It is not part /// of the content-addressed header identity. @@ -690,11 +687,9 @@ impl RbcInitialProof { BlockAuthentication::MlDsa44(signature) => Ok(Self::MlDsa44(signature.clone())), BlockAuthentication::MlDsa65(signature) => Ok(Self::MlDsa65(signature.clone())), BlockAuthentication::MacTag(tag) => Ok(Self::Mac(*tag)), - BlockAuthentication::MacVector(tags) if !tags.is_empty() => { - Ok(Self::MacVector(tags.clone())) + BlockAuthentication::None | BlockAuthentication::MacVector(_) => { + Err(RbcError::InvalidInitialProof) } - BlockAuthentication::MacVector(_) => Err(RbcError::InvalidInitialProof), - BlockAuthentication::None => Err(RbcError::InvalidInitialProof), } } } @@ -918,6 +913,7 @@ pub(crate) enum RbcEffect { holders: AuthoritySet, }, Deliver(PinnedRbcHeader), + PortableEchoVote(BlockReference), PortableEchoQc(StarfishRbcEchoQcV3), } @@ -985,6 +981,7 @@ pub(crate) enum RbcError { DuplicateAcknowledgment(BlockReference), InvalidThresholdClock, InvalidSingleDagEvidence, + InvalidPortableEchoVote, InvalidPortableEchoQc, HeaderDigestMismatch { expected: BlockDigest, @@ -1127,6 +1124,9 @@ impl fmt::Display for RbcError { Self::InvalidSingleDagEvidence => { f.write_str("Starfish-RBC V3 block carries non-canonical reference evidence") } + Self::InvalidPortableEchoVote => { + f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO vote") + } Self::InvalidPortableEchoQc => { f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO-QC") } @@ -1212,12 +1212,8 @@ struct CandidateState { ready_validity_observed: bool, ready_quorum_observed: bool, header_request_holders: AuthoritySet, - /// Exact authenticated ordinary blocks whose authors carried - /// `ECHO(this candidate)`. These references are the signature-free QC - /// witnesses; no standalone vote object exists. - portable_echo_witnesses: BTreeMap, - pending_portable_echo_qc: Option, - portable_echo_qc_holders: AuthoritySet, + portable_echo_votes: BTreeMap, + portable_echo_votes_verified: bool, portable_echo_qc_emitted: bool, portable_echo_qc_observed: bool, } @@ -1232,9 +1228,8 @@ impl CandidateState { ready_validity_observed: false, ready_quorum_observed: false, header_request_holders: AuthoritySet::default(), - portable_echo_witnesses: BTreeMap::new(), - pending_portable_echo_qc: None, - portable_echo_qc_holders: AuthoritySet::default(), + portable_echo_votes: BTreeMap::new(), + portable_echo_votes_verified: false, portable_echo_qc_emitted: false, portable_echo_qc_observed: false, } @@ -1300,7 +1295,7 @@ enum ProgressAction { NeedHeader(AuthoritySet), SendReady, Deliver, - EmitPortableEchoQc(Vec), + EmitPortableEchoQc(Vec), None, } @@ -1311,14 +1306,10 @@ pub(crate) struct StarfishRbcKernel { mac_keys: Arc>, local_round: RoundNumber, minimum_new_slot_round: RoundNumber, - /// Testbed portable fast path. Exact MAC-vector-authenticated ordinary - /// blocks carrying ECHO are named by a quorum certificate in a later - /// ordinary single-DAG block. + /// Testbed portable fast path. Public ECHO signatures are copied into a + /// quorum certificate carried by an ordinary single-DAG block. echo_qc_fast_path: bool, slots: BTreeMap>, - /// Reverse dependency index for QCs received before every exact witness - /// block is locally authenticated. Each set is committee-bounded. - portable_qc_waiters: BTreeMap>, } impl StarfishRbcKernel { @@ -1370,7 +1361,6 @@ impl StarfishRbcKernel { minimum_new_slot_round: 1, echo_qc_fast_path, slots: BTreeMap::new(), - portable_qc_waiters: BTreeMap::new(), }) } @@ -1444,8 +1434,7 @@ impl StarfishRbcKernel { header.ensure_committee(self.context.committee_id)?; let block_ref = header.reference(); self.validate_block_ref(&block_ref)?; - let portable_mac_vector = matches!(proof, RbcInitialProof::MacVector(_)); - if direct_peer != block_ref.authority && !portable_mac_vector { + if direct_peer != block_ref.authority { return Err(RbcError::InitialAuthorMismatch { expected: block_ref.authority, actual: direct_peer, @@ -1489,9 +1478,6 @@ impl StarfishRbcKernel { (BlockAuthenticationScheme::MacVector, RbcInitialProof::Mac(tag)) => { self.verify_initial_mac_tag(direct_peer, block_ref, tag)?; } - (BlockAuthenticationScheme::MacVector, RbcInitialProof::MacVector(tags)) => { - self.verify_initial_mac_vector(block_ref, tags)?; - } _ => return Err(RbcError::InitialProofSchemeMismatch), } Ok(EchoEligibleHeader { @@ -1611,7 +1597,7 @@ impl StarfishRbcKernel { ) -> Result { let pinned = self.validate_header_content(header)?; let block_ref = pinned.reference(); - if direct_peer != block_ref.authority && !matches!(proof, RbcInitialProof::MacVector(_)) { + if direct_peer != block_ref.authority { return Err(RbcError::InitialAuthorMismatch { expected: block_ref.authority, actual: direct_peer, @@ -1693,12 +1679,15 @@ impl StarfishRbcKernel { .echoes .add(own_authority, &committee); - let mut effects = vec![RbcEffect::MulticastPhase { - phase: RbcPhase::Echo, - block_ref, + let mut effects = vec![if self.echo_qc_fast_path { + RbcEffect::PortableEchoVote(block_ref) + } else { + RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, + } }]; effects.extend(self.drive(block_ref)); - effects.extend(self.retry_portable_qcs_waiting_on(block_ref)); Ok(effects) } @@ -1734,43 +1723,21 @@ impl StarfishRbcKernel { pub(crate) fn handle_embedded_reference( &mut self, authenticated_sender: AuthorityIndex, - enclosing_block: BlockReference, evidence: StarfishRbcReferenceV3, ) -> Result, RbcError> { if !self.committee.known_authority(authenticated_sender) { return Err(RbcError::UnknownAuthority(authenticated_sender)); } - if enclosing_block.authority != authenticated_sender { - return Err(RbcError::InvalidPortableEchoQc); - } let block_ref = evidence.reference(); self.validate_block_ref(&block_ref)?; let phase = match evidence.kind() { StarfishRbcReferenceKindV3::Echo => RbcPhase::Echo, StarfishRbcReferenceKindV3::Ready => RbcPhase::Ready, }; - let echo_qc_fast_path = self.echo_qc_fast_path; let committee = Arc::clone(&self.committee); let slot = self.slot_mut(block_ref); - if phase == RbcPhase::Echo && echo_qc_fast_path { - let candidate = slot - .candidates - .entry(block_ref) - .or_insert_with(CandidateState::new); - match candidate.portable_echo_witnesses.get(&authenticated_sender) { - Some(existing) if *existing != enclosing_block => { - return Err(RbcError::InvalidPortableEchoQc); - } - Some(_) => {} - None => { - candidate - .portable_echo_witnesses - .insert(authenticated_sender, enclosing_block); - } - } - } if !slot.record_phase_sender(phase, authenticated_sender, block_ref) { - return Ok(self.drive(block_ref)); + return Ok(Vec::new()); } let candidate = slot .candidates @@ -1787,17 +1754,127 @@ impl StarfishRbcKernel { Ok(self.drive(block_ref)) } + /// Digest signed by an ECHO sender for the portable single-DAG fast path. + /// The protocol instance and committee identifier prevent cross-run reuse. + pub(crate) fn portable_echo_signature_digest( + &self, + target: BlockReference, + ) -> Result<[u8; 32], RbcError> { + self.validate_block_ref(&target)?; + Ok(blake3::hash(&encode_base_statement( + &self.context, + PORTABLE_ECHO_KIND, + &target, + )) + .into()) + } + + pub(crate) fn handle_portable_echo_vote( + &mut self, + vote: StarfishRbcEchoVoteV3, + ) -> Result, RbcError> { + if !self.echo_qc_fast_path { + return Err(RbcError::InvalidPortableEchoVote); + } + self.validate_block_ref(&vote.target())?; + if self + .candidate(&vote.target()) + .is_some_and(|candidate| candidate.portable_echo_votes_verified) + { + return Ok(Vec::new()); + } + let candidate = self.candidate_mut(vote.target()); + match candidate.portable_echo_votes.get(&vote.sender()) { + Some(existing) if existing != &vote => return Err(RbcError::InvalidPortableEchoVote), + Some(_) => return Ok(Vec::new()), + None => { + candidate.portable_echo_votes.insert(vote.sender(), vote); + } + } + self.verify_portable_echo_vote_batch(vote.target())?; + Ok(self.drive(vote.target())) + } + + fn verify_portable_echo_vote_batch(&mut self, target: BlockReference) -> Result<(), RbcError> { + let votes: Vec<_> = self + .candidate(&target) + .into_iter() + .flat_map(|candidate| candidate.portable_echo_votes.values().copied()) + .collect(); + let stake: Stake = votes + .iter() + .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) + .sum(); + if !self.committee.is_quorum(stake) { + return Ok(()); + } + let digest = self.portable_echo_signature_digest(target)?; + let batch_valid = |votes: &[StarfishRbcEchoVoteV3]| { + let mut signers = AuthoritySet::default(); + for vote in votes { + signers.insert(vote.sender()); + } + let signatures: Vec<_> = votes.iter().map(|vote| vote.signature()).collect(); + let signature_refs: Vec<_> = signatures.iter().collect(); + let Some(aggregate) = bls_try_aggregate(&signature_refs) else { + return false; + }; + let Some(public_keys) = bls_public_keys_for_signers(&self.committee, signers) else { + return false; + }; + bls_fast_aggregate_verify(&digest, &aggregate, &public_keys) + }; + let valid_votes = if batch_valid(&votes) { + votes + } else { + votes + .into_iter() + .filter(|vote| { + self.committee + .get_bls_public_key(vote.sender()) + .is_some_and(|public_key| { + public_key + .verify_trusted(&digest, &vote.signature()) + .is_ok() + }) + }) + .collect() + }; + let valid_stake: Stake = valid_votes + .iter() + .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) + .sum(); + let committee = Arc::clone(&self.committee); + let slot = self.slot_mut(target); + if committee.is_quorum(valid_stake) { + for vote in &valid_votes { + slot.record_phase_sender(RbcPhase::Echo, vote.sender(), target); + } + } + let candidate = slot + .candidates + .entry(target) + .or_insert_with(CandidateState::new); + candidate.portable_echo_votes = valid_votes + .iter() + .map(|vote| (vote.sender(), *vote)) + .collect(); + if committee.is_quorum(valid_stake) { + candidate.portable_echo_votes_verified = true; + for vote in valid_votes { + candidate.echoes.add(vote.sender(), &committee); + } + } + Ok(()) + } + pub(crate) fn handle_portable_echo_qc( &mut self, - authenticated_sender: AuthorityIndex, qc: &StarfishRbcEchoQcV3, ) -> Result, RbcError> { if !self.echo_qc_fast_path { return Err(RbcError::InvalidPortableEchoQc); } - if !self.committee.known_authority(authenticated_sender) { - return Err(RbcError::UnknownAuthority(authenticated_sender)); - } self.validate_block_ref(&qc.target())?; if self .candidate(&qc.target()) @@ -1805,139 +1882,45 @@ impl StarfishRbcKernel { { return Ok(self.drive(qc.target())); } - self.validate_portable_echo_qc_shape(qc)?; - let candidate = self.candidate_mut(qc.target()); - candidate - .portable_echo_qc_holders - .insert(authenticated_sender); - match candidate.pending_portable_echo_qc.as_ref() { - Some(existing) if existing <= qc => {} - _ => candidate.pending_portable_echo_qc = Some(qc.clone()), - } - Ok(self.try_complete_portable_echo_qc(qc.target())) - } - - fn validate_portable_echo_qc_shape(&self, qc: &StarfishRbcEchoQcV3) -> Result<(), RbcError> { - if qc.witnesses().is_empty() - || qc.witnesses().len() > self.committee.len() - || qc.witnesses().windows(2).any(|pair| pair[0] >= pair[1]) - { - return Err(RbcError::InvalidPortableEchoQc); - } - let mut authors = AuthoritySet::default(); let mut stake = 0; - for witness in qc.witnesses() { - self.validate_block_ref(witness)?; - if witness.round <= qc.target().round || authors.contains(witness.authority) { - return Err(RbcError::InvalidPortableEchoQc); - } - authors.insert(witness.authority); + for sender in qc.signers().present() { stake += self .committee - .get_stake(witness.authority) + .get_stake(sender) .ok_or(RbcError::InvalidPortableEchoQc)?; } if !self.committee.is_quorum(stake) { return Err(RbcError::InvalidPortableEchoQc); } - Ok(()) - } - - fn authenticated_witness_carries_echo( - &self, - witness: BlockReference, - target: BlockReference, - ) -> Option { - let slot = self.slot(&witness)?; - if slot.echoed != Some(witness) { - return None; - } - let header = slot.candidates.get(&witness)?.header.as_ref()?; - Some(header.header().starfish_rbc_v3().is_some_and(|fields| { - fields - .references() - .binary_search(&StarfishRbcReferenceV3::new( - StarfishRbcReferenceKindV3::Echo, - target, - )) - .is_ok() - })) - } - - fn try_complete_portable_echo_qc(&mut self, target: BlockReference) -> Vec { - let Some((qc, holders)) = self.candidate(&target).and_then(|candidate| { - candidate - .pending_portable_echo_qc - .clone() - .map(|qc| (qc, candidate.portable_echo_qc_holders)) - }) else { - return Vec::new(); - }; - - let mut missing = Vec::new(); - for witness in qc.witnesses() { - match self.authenticated_witness_carries_echo(*witness, target) { - Some(true) => {} - Some(false) => { - self.candidate_mut(target).pending_portable_echo_qc = None; - return Vec::new(); - } - None => missing.push(*witness), - } - } - if !missing.is_empty() { - for witness in &missing { - self.portable_qc_waiters - .entry(*witness) - .or_default() - .insert(target); - } - return missing - .into_iter() - .map(|block_ref| RbcEffect::NeedHeader { block_ref, holders }) - .collect(); - } - - for witness in qc.witnesses() { - if let Some(waiters) = self.portable_qc_waiters.get_mut(witness) { - waiters.remove(&target); - if waiters.is_empty() { - self.portable_qc_waiters.remove(witness); - } - } + let public_keys = bls_public_keys_for_signers(&self.committee, qc.signers()) + .ok_or(RbcError::InvalidPortableEchoQc)?; + if !bls_fast_aggregate_verify( + &self.portable_echo_signature_digest(qc.target())?, + &qc.signature(), + &public_keys, + ) { + return Err(RbcError::InvalidPortableEchoQc); } - let candidate = self.candidate_mut(target); - candidate.pending_portable_echo_qc = None; + let candidate = self.candidate_mut(qc.target()); candidate.portable_echo_qc_observed = true; let relay = if candidate.portable_echo_qc_emitted { None } else { candidate.portable_echo_qc_emitted = true; - Some(RbcEffect::PortableEchoQc(qc)) + Some(RbcEffect::PortableEchoQc(qc.clone())) }; let mut effects = relay.into_iter().collect::>(); - effects.extend(self.drive(target)); + effects.extend(self.drive(qc.target())); if self - .candidate(&target) + .candidate(&qc.target()) .is_some_and(|candidate| candidate.header.is_none()) { effects.push(RbcEffect::NeedHeader { - block_ref: target, - holders, + block_ref: qc.target(), + holders: qc.signers(), }); } - effects - } - - fn retry_portable_qcs_waiting_on(&mut self, witness: BlockReference) -> Vec { - let targets = self - .portable_qc_waiters - .remove(&witness) - .unwrap_or_default(); - targets - .into_iter() - .flat_map(|target| self.try_complete_portable_echo_qc(target)) - .collect() + Ok(effects) } /// Materialize one recipient-specific message for an untagged multicast @@ -2037,26 +2020,6 @@ impl StarfishRbcKernel { self.make_initial_mac_tag_for_reference(block_ref, recipient) } - pub(crate) fn make_local_initial_mac_vector( - &self, - local: &RbcLocalInitial, - ) -> Result, RbcError> { - let block_ref = self.ensure_local_initial(local)?; - if self.context.initial_authentication != BlockAuthenticationScheme::MacVector { - return Err(RbcError::InitialMacRequiresMacAuthentication); - } - self.committee - .authorities() - .map(|recipient| { - if recipient == self.own_authority { - Ok(MacTag::from_bytes([0; crate::crypto::MAC_TAG_SIZE])) - } else { - self.make_initial_mac_tag_for_reference(block_ref, recipient) - } - }) - .collect() - } - fn make_initial_mac_tag_for_reference( &self, block_ref: BlockReference, @@ -2122,35 +2085,6 @@ impl StarfishRbcKernel { Ok(()) } - pub(crate) fn verify_initial_mac_vector( - &self, - block_ref: BlockReference, - tags: &[MacTag], - ) -> Result<(), RbcError> { - if self.context.initial_authentication != BlockAuthenticationScheme::MacVector { - return Err(RbcError::InitialMacRequiresMacAuthentication); - } - self.validate_block_ref(&block_ref)?; - if tags.len() != self.committee.len() { - return Err(RbcError::InvalidInitialProof); - } - if block_ref.authority == self.own_authority { - return Err(RbcError::LoopbackPhase); - } - let statement = encode_mac_statement( - &self.context, - INITIAL_KIND, - &block_ref, - block_ref.authority, - self.own_authority, - ); - let expected = self.mac_keys[block_ref.authority as usize].compute_rbc_tag(&statement); - if tags[self.own_authority as usize] != expected { - return Err(RbcError::InvalidInitialTag); - } - Ok(()) - } - #[allow(dead_code)] pub(crate) fn header_holders(&self, block_ref: &BlockReference) -> AuthoritySet { self.candidate(block_ref) @@ -2326,14 +2260,15 @@ impl StarfishRbcKernel { let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; - let portable_witness_stake = candidate - .portable_echo_witnesses + let portable_vote_stake = candidate + .portable_echo_votes .keys() .map(|sender| committee.get_stake(*sender).unwrap_or_default()) .sum::(); let portable_qc_ready = echo_qc_fast_path && !candidate.portable_echo_qc_emitted - && committee.is_quorum(portable_witness_stake); + && candidate.portable_echo_votes_verified + && committee.is_quorum(portable_vote_stake); let blocked_on_header = candidate.header.is_none() && ((can_send_ready && ready_trigger) || (can_deliver @@ -2342,11 +2277,7 @@ impl StarfishRbcKernel { let holders = candidate.holders(); if portable_qc_ready { ProgressAction::EmitPortableEchoQc( - candidate - .portable_echo_witnesses - .values() - .copied() - .collect(), + candidate.portable_echo_votes.values().copied().collect(), ) } else if blocked_on_header && holders != candidate.header_request_holders { candidate.header_request_holders = holders; @@ -2404,7 +2335,7 @@ impl StarfishRbcKernel { effects.push(RbcEffect::Deliver(header)); } } - ProgressAction::EmitPortableEchoQc(witnesses) => { + ProgressAction::EmitPortableEchoQc(votes) => { let candidate = self.candidate_mut(block_ref); if !candidate.portable_echo_qc_emitted { candidate.portable_echo_qc_emitted = true; @@ -2413,8 +2344,17 @@ impl StarfishRbcKernel { // creation/dissemination of the QC-bearing block, so // delivery cannot overtake portable publication. candidate.portable_echo_qc_observed = true; + let mut signers = AuthoritySet::default(); + for vote in &votes { + signers.insert(vote.sender()); + } + let signatures: Vec<_> = + votes.iter().map(|vote| vote.signature()).collect(); + let signature_refs: Vec<_> = signatures.iter().collect(); effects.push(RbcEffect::PortableEchoQc(StarfishRbcEchoQcV3::new( - block_ref, witnesses, + block_ref, + signers, + bls_aggregate(&signature_refs), ))); } } @@ -2527,7 +2467,8 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, + mac_keyrings_for_test, }, types::{BlockDigest, BlockReference}, }; @@ -2598,13 +2539,11 @@ mod tests { let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); assert!( receiver - .handle_embedded_reference(1, block(1, 2, 0x81), echo) + .handle_embedded_reference(1, echo) .unwrap() .is_empty() ); - let effects = receiver - .handle_embedded_reference(2, block(2, 2, 0x82), echo) - .unwrap(); + let effects = receiver.handle_embedded_reference(2, echo).unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::MulticastPhase { @@ -2621,13 +2560,11 @@ mod tests { let ready = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Ready, target); assert!( receiver - .handle_embedded_reference(1, block(1, 3, 0x91), ready) + .handle_embedded_reference(1, ready) .unwrap() .is_empty() ); - let effects = receiver - .handle_embedded_reference(2, block(2, 3, 0x92), ready) - .unwrap(); + let effects = receiver.handle_embedded_reference(2, ready).unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::Deliver(header) if header.reference() == target @@ -2635,7 +2572,7 @@ mod tests { } #[test] - fn portable_echo_qc_fast_path_uses_exact_authenticated_dag_witnesses() { + fn portable_echo_qc_fast_path_requires_and_accepts_exact_signed_quorum() { let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(committee.len()); let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( @@ -2654,32 +2591,21 @@ mod tests { .unwrap(); receiver.authorize_echo(target).unwrap(); - let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); - let witnesses: Vec<_> = (0..3) - .map(|sender| block(sender, 2, 0x80 + sender as u8)) + let digest = receiver.portable_echo_signature_digest(target).unwrap(); + let signer = dummy_bls_signer(); + let votes: Vec<_> = (0..3) + .map(|sender| StarfishRbcEchoVoteV3::new(target, sender, signer.sign_digest(&digest))) .collect(); - let mut effects = Vec::new(); - for witness in &witnesses { - let header = PinnedRbcHeader { - header: Arc::new(RbcCanonicalHeader { - reference: *witness, - block_references: Vec::new(), - acknowledgments: RbcAckFields { - intersection: Some(0), - extra_references: Vec::new(), - }, - meta_creation_time_ns: 0, - transactions_commitment: TransactionsCommitment::default(), - starfish_rbc_v3: Some(StarfishRbcFieldsV3::new(vec![echo])), - }), - committee_id: receiver.context.committee_id, - }; - receiver.note_header_available(header).unwrap(); - receiver.authorize_echo(*witness).unwrap(); - effects = receiver - .handle_embedded_reference(witness.authority, *witness, echo) - .unwrap(); + for vote in &votes[..2] { + assert!( + !receiver + .handle_portable_echo_vote(*vote) + .unwrap() + .iter() + .any(|effect| matches!(effect, RbcEffect::Deliver(_))) + ); } + let effects = receiver.handle_portable_echo_vote(votes[2]).unwrap(); assert!(!effects.iter().any(|effect| matches!( effect, @@ -2694,8 +2620,7 @@ mod tests { RbcEffect::PortableEchoQc(qc) => Some(qc.clone()), _ => None, }) - .expect("exact witness quorum must emit a portable ECHO-QC"); - assert_eq!(qc.witnesses(), witnesses); + .expect("signed quorum must emit a portable ECHO-QC"); let qc_position = effects .iter() .position(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) @@ -2710,7 +2635,7 @@ mod tests { }) .expect("delivery follows portable publication"); assert!(qc_position < delivery_position); - assert!(receiver.handle_portable_echo_qc(1, &qc).unwrap().is_empty()); + assert!(receiver.handle_portable_echo_qc(&qc).unwrap().is_empty()); let committee = Committee::new_test(vec![1; 4]); let keyrings = mac_keyrings_for_test(committee.len()); @@ -2724,52 +2649,77 @@ mod tests { true, ) .unwrap(); - late_receiver - .note_header_available(pinned_header_for_context(late_receiver.context, target)) - .unwrap(); - let missing = late_receiver.handle_portable_echo_qc(1, &qc).unwrap(); - assert_eq!( - missing - .iter() - .filter(|effect| matches!(effect, RbcEffect::NeedHeader { .. })) - .count(), - witnesses.len() - ); - let mut effects = Vec::new(); - for witness in &witnesses { - let header = PinnedRbcHeader { - header: Arc::new(RbcCanonicalHeader { - reference: *witness, - block_references: Vec::new(), - acknowledgments: RbcAckFields { - intersection: Some(0), - extra_references: Vec::new(), - }, - meta_creation_time_ns: 0, - transactions_commitment: TransactionsCommitment::default(), - starfish_rbc_v3: Some(StarfishRbcFieldsV3::new(vec![echo])), - }), - committee_id: late_receiver.context.committee_id, - }; - late_receiver.note_header_available(header).unwrap(); - effects = late_receiver.authorize_echo(*witness).unwrap(); - } - assert!(effects.iter().any(|effect| matches!( + let missing = late_receiver.handle_portable_echo_qc(&qc).unwrap(); + assert!(missing.iter().any(|effect| matches!( effect, RbcEffect::PortableEchoQc(relayed) if relayed == &qc ))); + assert!(missing.iter().any(|effect| matches!( + effect, + RbcEffect::NeedHeader { block_ref, .. } if *block_ref == target + ))); + let effects = late_receiver + .note_header_available(pinned_header_for_context(late_receiver.context, target)) + .unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::Deliver(header) if header.reference() == target ))); let other_target = block(3, 1, 0x73); - let forged = StarfishRbcEchoQcV3::new(other_target, witnesses); - let effects = receiver.handle_portable_echo_qc(1, &forged).unwrap(); + let forged = StarfishRbcEchoQcV3::new(other_target, qc.signers(), qc.signature()); + assert_eq!( + receiver.handle_portable_echo_qc(&forged), + Err(RbcError::InvalidPortableEchoQc) + ); + } + + #[test] + fn invalid_echo_vote_cannot_poison_a_later_portable_quorum() { + let committee = Committee::new_test(vec![1; 4]); + let keyrings = mac_keyrings_for_test(committee.len()); + let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( + committee, + 0, + instance(TEST_INSTANCE_BYTE), + BlockAuthenticationScheme::MacVector, + Arc::new(keyrings[0].clone()), + 1, + true, + ) + .unwrap(); + let target = block(3, 1, 0x74); + receiver + .note_header_available(pinned_header_for_context(receiver.context, target)) + .unwrap(); + let digest = receiver.portable_echo_signature_digest(target).unwrap(); + let signer = dummy_bls_signer(); + let wrong_signature = signer.sign_digest(&[0xFF; 32]); + for (sender, signature) in [ + (0, signer.sign_digest(&digest)), + (1, wrong_signature), + (2, signer.sign_digest(&digest)), + ] { + let effects = receiver + .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new(target, sender, signature)) + .unwrap(); + assert!( + !effects + .iter() + .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) + ); + } + let effects = receiver + .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new( + target, + 3, + signer.sign_digest(&digest), + )) + .unwrap(); assert!( - !effects + effects .iter() - .any(|effect| matches!(effect, RbcEffect::Deliver(_))) + .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) ); } @@ -2961,7 +2911,7 @@ mod tests { ); deliveries[owner as usize].push(header.reference()); } - RbcEffect::PortableEchoQc(_) => { + RbcEffect::PortableEchoVote(_) | RbcEffect::PortableEchoQc(_) => { panic!("portable ECHO effects require the single-DAG test harness") } } @@ -3112,14 +3062,19 @@ mod tests { } #[test] - fn single_dag_digest_binds_portable_echo_witness_certificate() { + fn single_dag_digest_binds_portable_echo_votes_and_certificate() { let target = block(0, 4, 0x43); - let qc = StarfishRbcEchoQcV3::new( - target, - vec![block(0, 5, 0x44), block(1, 5, 0x45), block(2, 5, 0x46)], - ); + let signature = dummy_bls_signer().sign_digest(&[0x44; 32]); + let vote = StarfishRbcEchoVoteV3::new(target, 1, signature); + let mut signers = AuthoritySet::default(); + signers.insert(0); + signers.insert(1); + signers.insert(2); + let signatures = [&signature, &signature, &signature]; + let qc = StarfishRbcEchoQcV3::new(target, signers, bls_aggregate(&signatures)); let empty = StarfishRbcFieldsV3::default(); - let with_qc = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), vec![qc]); + let with_vote = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), vec![vote], Vec::new()); + let with_qc = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), Vec::new(), vec![qc]); let digest = |fields: &StarfishRbcFieldsV3| { BlockDigest::new_starfish_rbc_single_dag_header( 3, @@ -3131,7 +3086,9 @@ mod tests { fields, ) }; + assert_ne!(digest(&empty), digest(&with_vote)); assert_ne!(digest(&empty), digest(&with_qc)); + assert_ne!(digest(&with_vote), digest(&with_qc)); } #[test] @@ -4015,66 +3972,6 @@ mod tests { )); } - #[test] - fn complete_initial_mac_vector_is_relayable_and_receiver_verifiable() { - let committee = Committee::new_test(vec![1; 4]); - let keyrings = mac_keyrings_for_test(4); - let template = valid_canonical_header(0, 7, 0x45); - let mut author = kernel( - Arc::clone(&committee), - &keyrings, - 0, - BlockAuthenticationScheme::MacVector, - ); - let local = author - .start_local_initial_header( - template.reference().round, - template.block_references().to_vec(), - template.acknowledgment_references(), - template.meta_creation_time_ns(), - template.transactions_commitment(), - ) - .unwrap(); - let canonical = local.header().clone(); - let vector = author.make_local_initial_mac_vector(&local).unwrap(); - assert_eq!(vector.len(), committee.len()); - - let mut receiver = kernel( - Arc::clone(&committee), - &keyrings, - 2, - BlockAuthenticationScheme::MacVector, - ); - assert!(matches!( - receiver - .accept_direct_initial_header( - 1, - canonical.clone(), - &RbcInitialProof::MacVector(vector.clone()), - ) - .unwrap(), - RbcInitialHeaderOutcome::Authenticated { .. } - )); - - let mut poisoned = vector; - poisoned[2] = MacTag::from_bytes([0; crate::crypto::MAC_TAG_SIZE]); - let mut other_receiver = kernel( - committee, - &keyrings, - 2, - BlockAuthenticationScheme::MacVector, - ); - assert!(matches!( - other_receiver - .accept_direct_initial_header(1, canonical, &RbcInitialProof::MacVector(poisoned),) - .unwrap(), - RbcInitialHeaderOutcome::StagedUnauthenticated { - error: RbcError::InvalidInitialTag, - .. - } - )); - } - #[test] fn signature_digest_binds_context_scheme_and_reference() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index c6b234ef..90941860 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -25,8 +25,7 @@ use tokio::{ use crate::{ committee::Committee, - crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, - metrics::sample_starfish_rbc_single_dag_phase, + crypto::{BlsSigner, MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, network::NetworkMessage, starfish_rbc::{ PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, @@ -35,8 +34,8 @@ use crate::{ }, types::{ AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - RoundNumber, StarfishRbcEchoQcV3, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, - StarfishRbcReferenceV3, TimestampNs, TransactionData, + RoundNumber, StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, + StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, }, }; @@ -49,9 +48,7 @@ pub(crate) enum RbcInitialAuthenticator { Ed25519(Signer), MlDsa44(MlDsa44Signer), MlDsa65(MlDsa65Signer), - /// Signature-free initial authentication. The kernel derives the complete - /// receiver-verifiable MAC vector from its pairwise keyring. - Mac, + Mac(Signer, BlsSigner), } impl RbcInitialAuthenticator { @@ -60,7 +57,7 @@ impl RbcInitialAuthenticator { Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, - Self::Mac => BlockAuthenticationScheme::MacVector, + Self::Mac(_, _) => BlockAuthenticationScheme::MacVector, } } } @@ -162,10 +159,8 @@ pub(crate) enum RbcServiceEvent { Delivered(PinnedRbcHeader), /// An irrevocable local phase statement waiting to be embedded in the /// next ordinary Starfish block. - ReferenceReady { - reference: StarfishRbcReferenceV3, - target_creation_time_ns: Option, - }, + ReferenceReady(StarfishRbcReferenceV3), + EchoVoteReady(StarfishRbcEchoVoteV3), EchoQcReady(StarfishRbcEchoQcV3), Rejected { peer: Option, @@ -195,10 +190,6 @@ enum RbcServiceMessage { peer: AuthorityIndex, header: RbcCanonicalHeader, }, - HeaderEnvelopeResponse { - peer: AuthorityIndex, - proposal: RbcHeaderProposal, - }, PeerConnected(AuthorityIndex), PeerDisconnected(AuthorityIndex), #[allow(dead_code)] @@ -300,14 +291,6 @@ impl RbcServiceHandle { self.send(RbcServiceMessage::HeaderResponse { peer, header }) } - pub(crate) fn header_envelope_response( - &self, - peer: AuthorityIndex, - proposal: RbcHeaderProposal, - ) -> Result<(), RbcServiceError> { - self.send(RbcServiceMessage::HeaderEnvelopeResponse { peer, proposal }) - } - pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), RbcServiceError> { self.send(RbcServiceMessage::PeerConnected(peer)) } @@ -431,9 +414,7 @@ pub(crate) fn start_starfish_rbc_service_with_phase_authority( connected_peers: AuthoritySet::default(), pending_fetches: AHashMap::new(), staged_notifications: AHashSet::new(), - header_creation_times: AHashMap::new(), retained_initials: BTreeMap::new(), - retained_envelopes: AHashMap::new(), retained_phases: BTreeSet::new(), phase_authority, }; @@ -456,7 +437,14 @@ fn validate_local_authenticator( RbcInitialAuthenticator::MlDsa65(signer) => committee .get_ml_dsa_65_public_key(own_authority) .is_some_and(|public_key| public_key == &signer.public_key()), - RbcInitialAuthenticator::Mac => committee.known_authority(own_authority), + RbcInitialAuthenticator::Mac(signer, echo_signer) => { + committee + .get_public_key(own_authority) + .is_some_and(|public_key| public_key == &signer.public_key()) + && committee + .get_bls_public_key(own_authority) + .is_some_and(|public_key| public_key == &echo_signer.public_key()) + } }; if matches { Ok(()) @@ -503,16 +491,9 @@ struct RbcServiceState { connected_peers: AuthoritySet, pending_fetches: AHashMap, staged_notifications: AHashSet, - /// Diagnostic timestamp cache populated at the existing header-staging - /// boundary. A direct hash lookup avoids re-walking the kernel's nested - /// historical slot maps when ECHO or READY becomes locally eligible. - header_creation_times: AHashMap, /// Recipient-specialized local proposals retained for replay after a /// connection is replaced. Version one keeps these for the run. retained_initials: BTreeMap<(BlockReference, AuthorityIndex), RbcHeaderProposal>, - /// Authenticated full-vector proposal retained once per exact header for - /// signature-free witness recovery and relay. - retained_envelopes: AHashMap, /// Authorized local phase intents. Tags are rematerialized for the peer /// on replay rather than retaining or cloning a tagged wire message. retained_phases: BTreeSet<(BlockReference, RbcPhase)>, @@ -547,9 +528,6 @@ impl RbcServiceState { RbcServiceMessage::HeaderResponse { peer, header } => { self.accept_header_response(peer, header); } - RbcServiceMessage::HeaderEnvelopeResponse { peer, proposal } => { - self.accept_header_envelope_response(peer, proposal); - } RbcServiceMessage::PeerConnected(peer) => self.peer_connected(peer), RbcServiceMessage::PeerDisconnected(peer) => self.peer_disconnected(peer), RbcServiceMessage::AdvanceLocalRound { round, reply } => { @@ -593,13 +571,6 @@ impl RbcServiceState { let embedded_references = canonical.starfish_rbc_v3().cloned(); let transaction_data = transaction_data.map(Arc::new); let proposals = self.make_initial_proposals(&local, transaction_data); - if let Some((_, proposal)) = proposals - .first() - .filter(|(_, proposal)| matches!(proposal.proof(), RbcInitialProof::MacVector(_))) - { - self.retained_envelopes - .insert(canonical.reference(), proposal.clone()); - } let (pinned, effects) = local.into_parts(); self.notify_header_staged(pinned); @@ -609,11 +580,7 @@ impl RbcServiceState { self.send_network(recipient, NetworkMessage::RbcInitial(proposal)); } self.process_effects(effects); - self.process_embedded_references( - self.own_authority, - canonical.reference(), - embedded_references, - ); + self.process_embedded_references(self.own_authority, embedded_references); Ok(canonical) } @@ -650,52 +617,25 @@ impl RbcServiceState { RbcInitialProof::MlDsa65(signer.sign_digest(&BlockDigest::from(digest))); self.public_initial_proposals(header, proof, transaction_data) } - RbcInitialAuthenticator::Mac => { - if matches!( - self.phase_authority, - RbcPhaseAuthorityV1::EmbeddedSingleDag { - echo_qc_fast_path: true - } - ) { - let vector = self + RbcInitialAuthenticator::Mac(_, _) => self + .committee + .authorities() + .filter(|recipient| *recipient != self.own_authority) + .map(|recipient| { + let tag = self .kernel - .make_local_initial_mac_vector(local) + .make_local_initial_mac_tag(local, recipient) .expect("local RBC handle must remain selected"); - self.committee - .authorities() - .filter(|recipient| *recipient != self.own_authority) - .map(|recipient| { - ( - recipient, - RbcHeaderProposal::with_transaction_data( - header.clone(), - RbcInitialProof::MacVector(vector.clone()), - transaction_data.clone(), - ), - ) - }) - .collect() - } else { - self.committee - .authorities() - .filter(|recipient| *recipient != self.own_authority) - .map(|recipient| { - let tag = self - .kernel - .make_local_initial_mac_tag(local, recipient) - .expect("local RBC handle must remain selected"); - ( - recipient, - RbcHeaderProposal::with_transaction_data( - header.clone(), - RbcInitialProof::Mac(tag), - transaction_data.clone(), - ), - ) - }) - .collect() - } - } + ( + recipient, + RbcHeaderProposal::with_transaction_data( + header.clone(), + RbcInitialProof::Mac(tag), + transaction_data.clone(), + ), + ) + }) + .collect(), } } @@ -722,7 +662,6 @@ impl RbcServiceState { } fn accept_direct_initial(&mut self, peer: AuthorityIndex, proposal: RbcHeaderProposal) { - let retained = proposal.clone(); let (header, proof, transaction_data) = proposal.into_parts(); let block_ref = header.reference(); let embedded_references = header.starfish_rbc_v3().cloned(); @@ -731,32 +670,14 @@ impl RbcServiceState { .accept_direct_initial_header(peer, header, &proof) { Ok(RbcInitialHeaderOutcome::Authenticated { effects }) => { - if matches!(retained.proof(), RbcInitialProof::MacVector(_)) { - self.retained_envelopes.insert(block_ref, retained); - } let pinned = self.finish_header_staging(block_ref, Some(peer)); self.notify_transaction_payload(peer, pinned, transaction_data); self.process_effects(effects); - // A full MAC vector authenticates the canonical header's - // author even when `peer` is merely recovering/relaying it. - self.process_embedded_references( - block_ref.authority, - block_ref, - embedded_references, - ); + self.process_embedded_references(peer, embedded_references); } Ok(RbcInitialHeaderOutcome::StagedUnauthenticated { effects, error }) => { - // Preserve the content-addressed header for a later valid - // proof, but do not clear an exact recovery request or expose - // its payload as authoritative. Another advertised holder may - // still provide the receiver's valid MAC-vector entry. - match self.kernel.pinned_header(block_ref) { - Ok(Some(pinned)) => self.notify_header_staged(pinned), - Ok(None) => { - self.reject(Some(peer), RbcError::HeaderUnavailable(block_ref).into()) - } - Err(kernel_error) => self.reject(Some(peer), kernel_error.into()), - } + let pinned = self.finish_header_staging(block_ref, Some(peer)); + self.notify_transaction_payload(peer, pinned, transaction_data); self.process_effects(effects); self.reject(Some(peer), error.into()); } @@ -774,16 +695,10 @@ impl RbcServiceState { return; } match self.kernel.pinned_header(block_ref) { - Ok(Some(header)) => { - if let Some(proposal) = self.retained_envelopes.get(&block_ref).cloned() { - self.send_network(peer, NetworkMessage::RbcHeaderEnvelopeResponse(proposal)); - } else { - self.send_network( - peer, - NetworkMessage::RbcHeaderResponse(header.header().clone()), - ); - } - } + Ok(Some(header)) => self.send_network( + peer, + NetworkMessage::RbcHeaderResponse(header.header().clone()), + ), Ok(None) => {} Err(error) => self.reject(Some(peer), error.into()), } @@ -800,16 +715,6 @@ impl RbcServiceState { return; } let Some(fetch) = self.pending_fetches.get(&block_ref) else { - if self - .kernel - .pinned_header(block_ref) - .is_ok_and(|header| header.is_some()) - { - // A bounded fetch wave may have two holders in flight. Once - // the first exact response completes recovery, the second is - // an expected idempotent duplicate. - return; - } self.reject( Some(peer), RbcServiceError::UnexpectedHeaderResponse(block_ref), @@ -833,36 +738,6 @@ impl RbcServiceState { } } - fn accept_header_envelope_response( - &mut self, - peer: AuthorityIndex, - proposal: RbcHeaderProposal, - ) { - let block_ref = proposal.header().reference(); - let Some(fetch) = self.pending_fetches.get(&block_ref) else { - if self - .kernel - .pinned_header(block_ref) - .is_ok_and(|header| header.is_some()) - { - return; - } - self.reject( - Some(peer), - RbcServiceError::UnexpectedHeaderResponse(block_ref), - ); - return; - }; - if !fetch.holders.contains(peer) { - self.reject( - Some(peer), - RbcServiceError::HeaderResponseFromNonHolder { block_ref, peer }, - ); - return; - } - self.accept_direct_initial(peer, proposal); - } - fn finish_header_staging( &mut self, block_ref: BlockReference, @@ -901,10 +776,6 @@ impl RbcServiceState { } fn notify_header_staged(&mut self, header: PinnedRbcHeader) { - if sample_starfish_rbc_single_dag_phase(header.reference()) { - self.header_creation_times - .insert(header.reference(), header.header().meta_creation_time_ns()); - } if self.staged_notifications.insert(header.reference()) { let _ = self.events.send(RbcServiceEvent::HeaderStaged(header)); } @@ -925,14 +796,9 @@ impl RbcServiceState { RbcPhase::Echo => StarfishRbcReferenceKindV3::Echo, RbcPhase::Ready => StarfishRbcReferenceKindV3::Ready, }; - let target_creation_time_ns = - sample_starfish_rbc_single_dag_phase(block_ref) - .then(|| self.header_creation_times.get(&block_ref).copied()) - .flatten(); - let _ = self.events.send(RbcServiceEvent::ReferenceReady { - reference: StarfishRbcReferenceV3::new(kind, block_ref), - target_creation_time_ns, - }); + let _ = self.events.send(RbcServiceEvent::ReferenceReady( + StarfishRbcReferenceV3::new(kind, block_ref), + )); continue; } self.retained_phases.insert((block_ref, phase)); @@ -958,9 +824,26 @@ impl RbcServiceState { continue; } self.pending_fetches.remove(&header.reference()); - self.header_creation_times.remove(&header.reference()); let _ = self.events.send(RbcServiceEvent::Delivered(header)); } + RbcEffect::PortableEchoVote(target) => { + let RbcInitialAuthenticator::Mac(_, signer) = &self.initial_authenticator + else { + self.reject(None, RbcError::InvalidPortableEchoVote.into()); + continue; + }; + match self.kernel.portable_echo_signature_digest(target) { + Ok(digest) => { + let vote = StarfishRbcEchoVoteV3::new( + target, + self.own_authority, + signer.sign_digest(&digest), + ); + let _ = self.events.send(RbcServiceEvent::EchoVoteReady(vote)); + } + Err(error) => self.reject(None, error.into()), + } + } RbcEffect::PortableEchoQc(qc) => { let _ = self.events.send(RbcServiceEvent::EchoQcReady(qc)); } @@ -971,7 +854,6 @@ impl RbcServiceState { fn process_embedded_references( &mut self, sender: AuthorityIndex, - enclosing_block: BlockReference, references: Option, ) { if !matches!( @@ -988,16 +870,19 @@ impl RbcServiceState { return; }; for evidence in references.references() { - match self - .kernel - .handle_embedded_reference(sender, enclosing_block, *evidence) - { + match self.kernel.handle_embedded_reference(sender, *evidence) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(sender), error.into()), + } + } + for vote in references.echo_votes() { + match self.kernel.handle_portable_echo_vote(*vote) { Ok(effects) => self.process_effects(effects), Err(error) => self.reject(Some(sender), error.into()), } } for qc in references.echo_qcs() { - match self.kernel.handle_portable_echo_qc(sender, qc) { + match self.kernel.handle_portable_echo_qc(qc) { Ok(effects) => self.process_effects(effects), Err(error) => self.reject(Some(sender), error.into()), } @@ -1137,7 +1022,8 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, + dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, + mac_keyrings_for_test, }, starfish_rbc::RbcPhase, types::{TransactionData, VerifiedBlock}, @@ -1172,7 +1058,9 @@ mod tests { let keyrings = mac_keyrings_for_test(4); let authenticator = match scheme { BlockAuthenticationScheme::Ed25519 => RbcInitialAuthenticator::Ed25519(dummy_signer()), - BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, + BlockAuthenticationScheme::MacVector => { + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()) + } BlockAuthenticationScheme::MlDsa44 => { RbcInitialAuthenticator::MlDsa44(dummy_ml_dsa_44_signer()) } @@ -1206,7 +1094,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedCarrierDag, @@ -1227,7 +1115,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedSingleDag { @@ -1250,7 +1138,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedSingleDag { @@ -1384,17 +1272,9 @@ mod tests { message: NetworkMessage::RbcInitial(_), .. } => initials += 1, - RbcServiceEvent::ReferenceReady { - reference, - target_creation_time_ns, - } => { + RbcServiceEvent::ReferenceReady(reference) => { assert_eq!(reference.kind(), StarfishRbcReferenceKindV3::Echo); assert_eq!(reference.reference(), canonical.reference()); - assert_eq!( - target_creation_time_ns, - sample_starfish_rbc_single_dag_phase(canonical.reference()) - .then_some(canonical.meta_creation_time_ns()) - ); references += 1; } RbcServiceEvent::Network { @@ -1412,32 +1292,27 @@ mod tests { } #[tokio::test] - async fn portable_fast_path_emits_signature_free_echo_witness_without_phase_message() { + async fn portable_fast_path_emits_signed_echo_vote_without_phase_message() { let (handle, mut events, task) = start_single_dag_fast_service(); let mut header = local_header(1, 4); header.starfish_rbc_v3 = Some(StarfishRbcFieldsV3::default()); let canonical = handle.start_local_header(header).await.unwrap(); let mut initials = 0; - let mut echo_reference = None; + let mut vote = None; for _ in 0..5 { match next_event(&mut events).await { RbcServiceEvent::HeaderStaged(header) => { assert_eq!(header.reference(), canonical.reference()); } RbcServiceEvent::Network { - message: NetworkMessage::RbcInitial(proposal), + message: NetworkMessage::RbcInitial(_), .. - } => { - let RbcInitialProof::MacVector(tags) = proposal.proof() else { - panic!("portable MAC mode must retain the complete MAC vector") - }; - assert_eq!(tags.len(), 4); - initials += 1; - } - RbcServiceEvent::ReferenceReady { reference, .. } + } => initials += 1, + RbcServiceEvent::EchoVoteReady(echo_vote) => vote = Some(echo_vote), + RbcServiceEvent::ReferenceReady(reference) if reference.kind() == StarfishRbcReferenceKindV3::Echo => { - echo_reference = Some(reference); + panic!("portable mode emitted an unsigned ECHO reference") } RbcServiceEvent::Network { message: NetworkMessage::RbcPhase(_), @@ -1447,10 +1322,9 @@ mod tests { } } assert_eq!(initials, 3); - assert_eq!( - echo_reference.expect("portable mode must emit one embedded ECHO witness"), - StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, canonical.reference(),) - ); + let vote = vote.expect("portable mode must emit one signed ECHO vote"); + assert_eq!(vote.target(), canonical.reference()); + assert_eq!(vote.sender(), 0); drop(handle); task.await.unwrap(); } @@ -1627,7 +1501,7 @@ mod tests { }; assert_eq!(proposal.header(), &canonical); let RbcInitialProof::Mac(tag) = proposal.proof() else { - panic!("strict MAC mode must send one recipient tag") + panic!("MAC mode must send one tag") }; initial_proofs.push(*tag); } @@ -1852,7 +1726,7 @@ mod tests { instance(), BlockAuthenticationScheme::Ed25519, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, + RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), 1, Duration::from_secs(1), ); diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 004fb5b6..d5fd3f90 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -30,7 +30,7 @@ use crate::{ types::{ AuthorityIndex, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishNoVoteCert, SailfishTimeoutCert, Stake, - TimestampNs, VerifiedBlock, + VerifiedBlock, }, }; @@ -405,10 +405,13 @@ impl Syncer { pub fn apply_starfish_rbc_reference( &mut self, reference: crate::types::StarfishRbcReferenceV3, - target_creation_time_ns: Option, ) { - self.core - .add_starfish_rbc_reference(reference, target_creation_time_ns); + self.core.add_starfish_rbc_reference(reference); + self.try_new_block(BlockCreationReason::CertificateEvent); + } + + pub fn apply_starfish_rbc_echo_vote(&mut self, vote: crate::types::StarfishRbcEchoVoteV3) { + self.core.add_starfish_rbc_echo_vote(vote); self.try_new_block(BlockCreationReason::CertificateEvent); } diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 082e0af5..2afdb77a 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -138,29 +138,75 @@ pub struct StarfishRbcReferenceV3 { reference: BlockReference, } -/// Signature-free portable quorum certificate over exact ordinary-DAG blocks -/// that carry `ECHO(target)`. Every witness block is independently -/// authenticated with its author's complete MAC vector; a holder can relay -/// that vector and each receiver verifies only its own entry. +/// A publicly verifiable ECHO vote carried by an ordinary single-DAG block. +/// +/// The signature is deliberately independent of the carrying block. A later +/// block can therefore copy a quorum of votes into a portable certificate +/// without introducing a standalone RBC phase message. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub struct StarfishRbcEchoVoteV3 { + target: BlockReference, + sender: AuthorityIndex, + signature: BlsSignatureBytes, +} + +impl StarfishRbcEchoVoteV3 { + pub fn new( + target: BlockReference, + sender: AuthorityIndex, + signature: BlsSignatureBytes, + ) -> Self { + Self { + target, + sender, + signature, + } + } + + pub fn target(self) -> BlockReference { + self.target + } + + pub fn sender(self) -> AuthorityIndex { + self.sender + } + + pub fn signature(self) -> BlsSignatureBytes { + self.signature + } +} + +/// Portable quorum certificate over exact, publicly verifiable ECHO votes. #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct StarfishRbcEchoQcV3 { target: BlockReference, - witnesses: Vec, + signers: AuthoritySet, + signature: BlsSignatureBytes, } impl StarfishRbcEchoQcV3 { - pub fn new(target: BlockReference, mut witnesses: Vec) -> Self { - witnesses.sort_unstable(); - witnesses.dedup(); - Self { target, witnesses } + pub fn new( + target: BlockReference, + signers: AuthoritySet, + signature: BlsSignatureBytes, + ) -> Self { + Self { + target, + signers, + signature, + } } pub fn target(&self) -> BlockReference { self.target } - pub fn witnesses(&self) -> &[BlockReference] { - &self.witnesses + pub fn signers(&self) -> AuthoritySet { + self.signers + } + + pub fn signature(&self) -> BlsSignatureBytes { + self.signature } } @@ -187,6 +233,8 @@ impl StarfishRbcReferenceV3 { pub struct StarfishRbcFieldsV3 { references: Vec, #[serde(default)] + echo_votes: Vec, + #[serde(default)] echo_qcs: Vec, } @@ -196,20 +244,25 @@ impl StarfishRbcFieldsV3 { references.dedup(); Self { references, + echo_votes: Vec::new(), echo_qcs: Vec::new(), } } pub fn with_portable_echo( mut references: Vec, + mut echo_votes: Vec, mut echo_qcs: Vec, ) -> Self { references.sort_unstable(); references.dedup(); + echo_votes.sort_unstable(); + echo_votes.dedup(); echo_qcs.sort_unstable(); echo_qcs.dedup(); Self { references, + echo_votes, echo_qcs, } } @@ -218,6 +271,10 @@ impl StarfishRbcFieldsV3 { &self.references } + pub fn echo_votes(&self) -> &[StarfishRbcEchoVoteV3] { + &self.echo_votes + } + pub fn echo_qcs(&self) -> &[StarfishRbcEchoQcV3] { &self.echo_qcs } @@ -230,7 +287,9 @@ impl StarfishRbcFieldsV3 { if self.references.len() > committee.len().saturating_mul(6) { return false; } - if self.echo_qcs.len() > committee.len() { + if self.echo_votes.len() > committee.len().saturating_mul(3) + || self.echo_qcs.len() > committee.len() + { return false; } if self.references.windows(2).any(|pair| pair[0] >= pair[1]) { @@ -244,34 +303,38 @@ impl StarfishRbcFieldsV3 { && committee.known_authority(reference.authority) && statements.insert((evidence.kind(), reference.authority, reference.round)) }); - if !references_valid || self.echo_qcs.windows(2).any(|pair| pair[0] >= pair[1]) { + if !references_valid + || self.echo_votes.windows(2).any(|pair| pair[0] >= pair[1]) + || self.echo_qcs.windows(2).any(|pair| pair[0] >= pair[1]) + { return false; } - self.echo_qcs.iter().all(|qc| { - let target = qc.target(); - if target.round == 0 - || target.round >= block_round - || !committee.known_authority(target.authority) - || qc.witnesses().is_empty() - || qc.witnesses().len() > committee.len() - || qc.witnesses().windows(2).any(|pair| pair[0] >= pair[1]) - { - return false; - } - let mut stake = 0; - let mut authors = AHashSet::new(); - for witness in qc.witnesses() { - if witness.round == 0 - || witness.round >= block_round - || !committee.known_authority(witness.authority) - || !authors.insert(witness.authority) + let votes_valid = self.echo_votes.iter().all(|vote| { + let target = vote.target(); + target.round > 0 + && target.round <= block_round + && committee.known_authority(target.authority) + && committee.known_authority(vote.sender()) + }); + votes_valid + && self.echo_qcs.iter().all(|qc| { + let target = qc.target(); + if target.round == 0 + || target.round >= block_round + || !committee.known_authority(target.authority) + || qc.signers().is_empty() { return false; } - stake += committee.get_stake(witness.authority).unwrap_or_default(); - } - committee.is_quorum(stake) - }) + let mut stake = 0; + for sender in qc.signers().present() { + if !committee.known_authority(sender) { + return false; + } + stake += committee.get_stake(sender).unwrap_or_default(); + } + committee.is_quorum(stake) + }) } } diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index ad65b8af..8a8f14ed 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -210,8 +210,8 @@ enum Operation { /// Requires embedded RBC-DAG authority and changes the finality proof. #[clap(long, default_value_t = false)] starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only: deliver a single-DAG RBC header from exact ordinary - /// DAG blocks carrying ECHO, authenticated only by MAC vectors. + /// Testbed-only: deliver a single-DAG RBC header from a portable + /// quorum of publicly signed ECHO votes carried in ordinary blocks. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. @@ -615,7 +615,7 @@ async fn local_benchmark( } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( - "Single-DAG MAC-witness ECHO-QC: ENABLED (signature-free; testbed-only totality assumption)" + "Single-DAG portable ECHO-QC delivery: ENABLED (signed votes; testbed evaluation)" ); } if let Some(latency) = node_parameters.uniform_latency_ms { diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index 2b26eb16..233a637b 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -37,8 +37,8 @@ is one of: The carrying block's authenticated author is the statement sender. The list is part of that ordinary block's content digest. Consequently the default path needs no standalone phase MAC, signature, or phase network message. The -flagged MAC-witness experiment adds exact references to ECHO-carrying ordinary -blocks, but still adds neither a signature nor a phase message. +portable fast path adds embedded BLS ECHO votes and an aggregate QC, but still +adds no phase message. References are sorted, duplicate-free and bounded by `6 * committee_size` per block. A sender may name at most one digest for each `(phase, target author, @@ -92,26 +92,26 @@ create a second physical round counter. ### Portable ECHO-QC fast path Finite benchmarks may opt into -`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode a certificate is the -target plus a quorum of exact ordinary-block references whose authenticated -headers contain `Echo(target)`. It contains no BLS, Ed25519, ML-DSA or other -digital signature. The certificate rides in an ordinary DAG block; there is no -standalone ECHO or READY message. - -For synchronization, fast-path MAC proposals retain the author's complete MAC -vector. A holder may relay the exact header and vector, and each receiver checks -only its own pairwise entry before accepting that witness. Normal delivery adds -no message; exact header/vector requests occur only when a named witness is -missing. - -This flag is deliberately a **latency lower bound, not the Byzantine-totality -mode**. A Byzantine witness author can make its vector valid for the first QC -builder but invalid for another receiver. Requiring every named quorum witness -therefore can prevent that receiver from accepting an otherwise observed QC; -accepting fewer witnesses would instead lose quorum-intersection safety. Public -signatures solve transferability but violate the signature-free requirement. -The default quorum-ECHO then quorum-READY path remains the signature-free, -totality-safe candidate. +`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode ECHO is a compact +BLS vote over the exact target reference, protocol instance and committee. +Votes ride in ordinary DAG blocks. Once quorum stake verifies, the node batch +aggregates them into one portable certificate containing the target, signer +bitmap and 48-byte aggregate signature. The QC also rides in an ordinary DAG +block; there is no standalone ECHO or READY message. + +An honest node publishes or relays the QC before its delivery effect. The +ordered Core bridge queues the QC-bearing block before applying delivery. +Quorum intersection gives uniqueness, while public verification and mandatory +relay remove the old receiver-local selective-withholding caveat: any valid QC +that reaches one honest validator can be verified and propagated by every +other validator. Missing target content still uses exact header recovery and +delivery remains fail-closed until that content is present. Invalid aggregate +batches fall back to individual vote verification so one Byzantine vote cannot +poison an otherwise valid quorum. + +The flag remains testbed-only because bounded retirement, durable pending-QC +replay and a complete asynchronous proof are still production follow-ups. The +default V3 path continues to use the conventional quorum-READY rule. ## Required validation @@ -133,16 +133,11 @@ with MAC authentication and the fixed 50 ms V3 round limiter: | `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | | V3 + old receiver-local ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | | V3 + old receiver-local ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | -| V3 + rejected BLS aggregate ECHO-QC | zero | 405.8 ms | 1,000 | 1.06 MB/s | -| V3 + rejected BLS aggregate ECHO-QC | AWS table | 983.7 ms | 1,000 | 0.74 MB/s | -| V3 + signature-free MAC-witness ECHO-QC | zero | 397.1 ms | 1,000 | 1.38 MB/s | -| V3 + signature-free MAC-witness ECHO-QC | AWS table | 955.3 ms | 1,000 | 0.91 MB/s | +| V3 + portable aggregate ECHO-QC | zero | 405.8 ms | 1,000 | 1.06 MB/s | +| V3 + portable aggregate ECHO-QC | AWS table | 983.7 ms | 1,000 | 0.74 MB/s | | `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | All exact offered transactions committed during the bounded drain. These are single-machine research measurements, not production claims. The two old -receiver-local rows preserve the historical totality caveat. The BLS rows are -retained only as historical latency data and are not part of the signature-free -design. The MAC-witness rows are also a flagged lower bound: unlike the strict -default path, they assume every named witness supplied a valid complete MAC -vector for every honest receiver. +receiver-local rows preserve the historical totality caveat; the new portable +rows use the signed aggregate certificate described above. From ee356170c7d2465221ab4163a338e46191e0a185 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:19:29 +0200 Subject: [PATCH 49/62] Revert "add portable single-DAG ECHO QC" This reverts commit 37379aa46cbb849fe5446be7a56ae9bc53dc4469. --- crates/starfish-core/src/config.rs | 7 +- crates/starfish-core/src/core.rs | 51 +- .../starfish-core/src/core_thread/spawned.rs | 35 -- crates/starfish-core/src/crypto.rs | 41 -- crates/starfish-core/src/net_sync.rs | 14 +- crates/starfish-core/src/starfish_rbc.rs | 457 ++---------------- .../starfish-core/src/starfish_rbc_service.rs | 130 +---- crates/starfish-core/src/syncer.rs | 10 - crates/starfish-core/src/types.rs | 153 +----- crates/starfish/src/main.rs | 7 +- docs/starfish-rbc-single-dag-v3.md | 49 +- 11 files changed, 74 insertions(+), 880 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 7701db4d..fad2d43e 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -94,9 +94,10 @@ pub struct NodeParameters { /// Starfish finality proof shape. #[serde(default)] pub starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed single-DAG RBC path: publicly signed ECHO votes are copied into - /// a portable quorum certificate carried by an ordinary DAG block. This - /// preserves uniqueness and totality without a standalone phase message. + /// Testbed-only single-DAG RBC path: deliver an exact header after quorum + /// ECHO rather than quorum READY. Quorum intersection preserves a unique + /// value, but selective Byzantine ECHO withholding can violate totality; + /// this must remain an explicit benchmark flag. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index fe8b6fc9..d2970e8b 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -39,8 +39,7 @@ use crate::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, RoundNumber, SailfishFields, Shard, - StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, StarfishRbcReferenceV3, - VerifiedBlock, + StarfishRbcFieldsV3, StarfishRbcReferenceV3, VerifiedBlock, }, }; @@ -58,8 +57,6 @@ pub struct Core { /// Irrevocable local ECHO/READY statements waiting to ride on the next /// ordinary block in the single-DAG protocol. pending_starfish_rbc_references: BTreeSet, - pending_starfish_rbc_echo_votes: BTreeSet, - pending_starfish_rbc_echo_qcs: BTreeSet, // For Byzantine node, last_own_block contains a vector of blocks last_own_block: Vec, block_handler: H, @@ -326,8 +323,6 @@ impl Core { pending, pending_reconstructed_data: AHashMap::new(), pending_starfish_rbc_references: BTreeSet::new(), - pending_starfish_rbc_echo_votes: BTreeSet::new(), - pending_starfish_rbc_echo_qcs: BTreeSet::new(), last_own_block: vec![last_own_block], block_handler, authority, @@ -363,10 +358,6 @@ impl Core { &self.signer } - pub fn get_bls_signer(&self) -> &BlsSigner { - &self.bls_signer - } - pub(crate) fn add_starfish_rbc_reference(&mut self, reference: StarfishRbcReferenceV3) { assert!( self.dag_state @@ -377,24 +368,6 @@ impl Core { self.pending_starfish_rbc_references.insert(reference); } - pub(crate) fn add_starfish_rbc_echo_vote(&mut self, vote: StarfishRbcEchoVoteV3) { - assert!( - self.dag_state - .consensus_protocol - .is_starfish_rbc_single_dag() - ); - self.pending_starfish_rbc_echo_votes.insert(vote); - } - - pub(crate) fn add_starfish_rbc_echo_qc(&mut self, qc: StarfishRbcEchoQcV3) { - assert!( - self.dag_state - .consensus_protocol - .is_starfish_rbc_single_dag() - ); - self.pending_starfish_rbc_echo_qcs.insert(qc); - } - pub(crate) fn get_ml_dsa_44_signer(&self) -> &crate::crypto::MlDsa44Signer { &self.ml_dsa_44_signer } @@ -884,27 +857,7 @@ impl Core { for reference in &references { self.pending_starfish_rbc_references.remove(reference); } - let echo_votes: Vec<_> = self - .pending_starfish_rbc_echo_votes - .iter() - .filter(|vote| vote.target().round <= clock_round) - .take(self.committee.len().saturating_mul(3)) - .copied() - .collect(); - for vote in &echo_votes { - self.pending_starfish_rbc_echo_votes.remove(vote); - } - let echo_qcs: Vec<_> = self - .pending_starfish_rbc_echo_qcs - .iter() - .filter(|qc| qc.target().round < clock_round) - .take(self.committee.len()) - .cloned() - .collect(); - for qc in &echo_qcs { - self.pending_starfish_rbc_echo_qcs.remove(qc); - } - StarfishRbcFieldsV3::with_portable_echo(references, echo_votes, echo_qcs) + StarfishRbcFieldsV3::new(references) }); // Create and store blocks diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 86728659..65405824 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -82,8 +82,6 @@ enum CoreThreadCommand { /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), - ApplyStarfishRbcEchoVote(crate::types::StarfishRbcEchoVoteV3, oneshot::Sender<()>), - ApplyStarfishRbcEchoQc(crate::types::StarfishRbcEchoQcV3, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. ApplyStarfishRbcDagFrontier( CommittedFrontierDeltaV1, @@ -287,23 +285,6 @@ impl CoreThread { self.syncer.apply_starfish_rbc_reference(reference); sender.send(()).ok(); } - CoreThreadCommand::ApplyStarfishRbcEchoVote(vote, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["apply_starfish_rbc_echo_vote"]) - .inc(); - self.syncer.apply_starfish_rbc_echo_vote(vote); - sender.send(()).ok(); - } - CoreThreadCommand::ApplyStarfishRbcEchoQc(qc, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["apply_starfish_rbc_echo_qc"]) - .inc(); - self.syncer.apply_starfish_rbc_echo_qc(qc); - sender.send(()).ok(); - } CoreThreadCommand::ApplyStarfishRbcDagFrontier(delta, sender) => { metrics .core_thread_tasks_total diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 27e38796..3d85e0bf 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -323,30 +323,6 @@ impl BlockDigest { hasher.update(&[evidence.kind().tag()]); hash_reference(&mut hasher, &evidence.reference()); } - // Preserve the frozen V3 digest exactly when the portable extension - // is absent. A non-empty extension is explicitly tagged before its - // length-delimited fields, so old stored/default blocks still reopen. - if !rbc.echo_votes().is_empty() || !rbc.echo_qcs().is_empty() { - hasher.update(b"PORTABLE_ECHO_QC_V1"); - let vote_len = u32::try_from(rbc.echo_votes().len()) - .expect("Starfish-RBC ECHO vote count exceeds u32"); - hasher.update(&vote_len.to_be_bytes()); - for vote in rbc.echo_votes() { - hash_reference(&mut hasher, &vote.target()); - hasher.update(&vote.sender().to_be_bytes()); - hasher.update(vote.signature().as_ref()); - } - let qc_len = u32::try_from(rbc.echo_qcs().len()) - .expect("Starfish-RBC ECHO-QC count exceeds u32"); - hasher.update(&qc_len.to_be_bytes()); - for qc in rbc.echo_qcs() { - hash_reference(&mut hasher, &qc.target()); - for word in qc.signers().words() { - hasher.update(&word.to_be_bytes()); - } - hasher.update(qc.signature().as_ref()); - } - } Self(hasher.finalize().into()) } @@ -1387,23 +1363,6 @@ pub fn bls_aggregate(sigs: &[&BlsSignatureBytes]) -> BlsSignatureBytes { BlsSignatureBytes(agg.to_signature().to_bytes()) } -/// Fallible counterpart used on untrusted wire votes before they have been -/// individually verified. The returned aggregate is still untrusted and must -/// be verified against the exact signer set and message. -pub fn bls_try_aggregate(sigs: &[&BlsSignatureBytes]) -> Option { - if sigs.is_empty() { - return None; - } - let parsed: Vec = sigs - .iter() - .map(|signature| bls::Signature::from_bytes(&signature.0)) - .collect::>() - .ok()?; - let references: Vec<_> = parsed.iter().collect(); - let aggregate = bls::AggregateSignature::aggregate(&references, true).ok()?; - Some(BlsSignatureBytes(aggregate.to_signature().to_bytes())) -} - /// Verify an aggregate signature against multiple public keys (all signed same /// message). #[allow(dead_code)] diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 1c95a86f..9f4ec5e7 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -3192,10 +3192,7 @@ impl NetworkSyncer BlockAuthenticationScheme::MlDsa65 => { RbcInitialAuthenticator::MlDsa65(core.get_ml_dsa_65_signer().clone()) } - BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac( - core.get_signer().clone(), - core.get_bls_signer().clone(), - ), + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, }; let (service, events, task) = start_starfish_rbc_service_with_phase_authority( committee.clone(), @@ -3571,15 +3568,6 @@ impl NetworkSyncer .apply_starfish_rbc_reference(reference) .await; } - RbcServiceEvent::EchoVoteReady(vote) => { - event_inner - .syncer - .apply_starfish_rbc_echo_vote(vote) - .await; - } - RbcServiceEvent::EchoQcReady(qc) => { - event_inner.syncer.apply_starfish_rbc_echo_qc(qc).await; - } RbcServiceEvent::Rejected { peer, error } => { tracing::warn!( "Rejected Starfish-RBC input from {:?}: {}", diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index feeb9446..4096a982 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -16,15 +16,13 @@ use crate::{ committee::{Committee, QuorumThreshold, StakeAggregator, ValidityThreshold}, crypto::{ Blake3Hasher, MacKey, MacTag, MlDsa44SignatureBytes, MlDsa65SignatureBytes, SignatureBytes, - TransactionsCommitment, bls_aggregate, bls_fast_aggregate_verify, - bls_public_keys_for_signers, bls_try_aggregate, + TransactionsCommitment, }, types::{ AckFields, AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockHeader, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, - StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, - StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, - VerifiedBlock, compress_acknowledgments, expand_acknowledgments, + StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, + TransactionData, VerifiedBlock, compress_acknowledgments, expand_acknowledgments, }, }; @@ -33,7 +31,6 @@ const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_V1_COMMITTEE_ID"; const INITIAL_KIND: u8 = 0x00; const ECHO_KIND: u8 = 0x01; const READY_KIND: u8 = 0x02; -const PORTABLE_ECHO_KIND: u8 = 0x03; const PROTOCOL_INSTANCE_SIZE: usize = 32; const COMMITTEE_ID_SIZE: usize = 32; @@ -526,27 +523,9 @@ impl RbcCanonicalHeader { ) }) .ok_or(RbcError::HeaderContentTooLarge)?; - let portable_echo_bytes = self.starfish_rbc_v3.as_ref().map_or(0, |rbc| { - let vote_bytes: usize = rbc - .echo_votes() - .iter() - .map(|vote| RBC_BLOCK_REFERENCE_SIZE + 2 + vote.signature().as_ref().len()) - .sum(); - let qc_bytes: usize = rbc - .echo_qcs() - .iter() - .map(|qc| { - RBC_BLOCK_REFERENCE_SIZE - + qc.signers().words().len() * std::mem::size_of::() - + qc.signature().as_ref().len() - }) - .sum(); - vote_bytes.saturating_add(qc_bytes) - }); RBC_BLOCK_REFERENCE_SIZE .checked_mul(reference_count) .and_then(|size| size.checked_add(RBC_HEADER_FIXED_CONTENT_SIZE)) - .and_then(|size| size.checked_add(portable_echo_bytes)) .ok_or(RbcError::HeaderContentTooLarge) } } @@ -913,8 +892,6 @@ pub(crate) enum RbcEffect { holders: AuthoritySet, }, Deliver(PinnedRbcHeader), - PortableEchoVote(BlockReference), - PortableEchoQc(StarfishRbcEchoQcV3), } #[derive(Clone, Debug, Eq, PartialEq)] @@ -981,8 +958,6 @@ pub(crate) enum RbcError { DuplicateAcknowledgment(BlockReference), InvalidThresholdClock, InvalidSingleDagEvidence, - InvalidPortableEchoVote, - InvalidPortableEchoQc, HeaderDigestMismatch { expected: BlockDigest, actual: BlockDigest, @@ -1124,12 +1099,6 @@ impl fmt::Display for RbcError { Self::InvalidSingleDagEvidence => { f.write_str("Starfish-RBC V3 block carries non-canonical reference evidence") } - Self::InvalidPortableEchoVote => { - f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO vote") - } - Self::InvalidPortableEchoQc => { - f.write_str("Starfish-RBC V3 block carries an invalid portable ECHO-QC") - } Self::HeaderDigestMismatch { expected, actual } => write!( f, "Starfish-RBC header digest mismatch: expected {expected}, got {actual}" @@ -1212,10 +1181,6 @@ struct CandidateState { ready_validity_observed: bool, ready_quorum_observed: bool, header_request_holders: AuthoritySet, - portable_echo_votes: BTreeMap, - portable_echo_votes_verified: bool, - portable_echo_qc_emitted: bool, - portable_echo_qc_observed: bool, } impl CandidateState { @@ -1228,10 +1193,6 @@ impl CandidateState { ready_validity_observed: false, ready_quorum_observed: false, header_request_holders: AuthoritySet::default(), - portable_echo_votes: BTreeMap::new(), - portable_echo_votes_verified: false, - portable_echo_qc_emitted: false, - portable_echo_qc_observed: false, } } @@ -1295,7 +1256,6 @@ enum ProgressAction { NeedHeader(AuthoritySet), SendReady, Deliver, - EmitPortableEchoQc(Vec), None, } @@ -1306,8 +1266,10 @@ pub(crate) struct StarfishRbcKernel { mac_keys: Arc>, local_round: RoundNumber, minimum_new_slot_round: RoundNumber, - /// Testbed portable fast path. Public ECHO signatures are copied into a - /// quorum certificate carried by an ordinary single-DAG block. + /// Testbed-only optimistic path. A quorum of locked ECHOs proves a unique + /// value, but without a portable proof it does not prove that every honest + /// node can assemble the same quorum under selective Byzantine + /// withholding. Keep disabled for the asynchronous RBC contract. echo_qc_fast_path: bool, slots: BTreeMap>, } @@ -1679,13 +1641,9 @@ impl StarfishRbcKernel { .echoes .add(own_authority, &committee); - let mut effects = vec![if self.echo_qc_fast_path { - RbcEffect::PortableEchoVote(block_ref) - } else { - RbcEffect::MulticastPhase { - phase: RbcPhase::Echo, - block_ref, - } + let mut effects = vec![RbcEffect::MulticastPhase { + phase: RbcPhase::Echo, + block_ref, }]; effects.extend(self.drive(block_ref)); Ok(effects) @@ -1754,175 +1712,6 @@ impl StarfishRbcKernel { Ok(self.drive(block_ref)) } - /// Digest signed by an ECHO sender for the portable single-DAG fast path. - /// The protocol instance and committee identifier prevent cross-run reuse. - pub(crate) fn portable_echo_signature_digest( - &self, - target: BlockReference, - ) -> Result<[u8; 32], RbcError> { - self.validate_block_ref(&target)?; - Ok(blake3::hash(&encode_base_statement( - &self.context, - PORTABLE_ECHO_KIND, - &target, - )) - .into()) - } - - pub(crate) fn handle_portable_echo_vote( - &mut self, - vote: StarfishRbcEchoVoteV3, - ) -> Result, RbcError> { - if !self.echo_qc_fast_path { - return Err(RbcError::InvalidPortableEchoVote); - } - self.validate_block_ref(&vote.target())?; - if self - .candidate(&vote.target()) - .is_some_and(|candidate| candidate.portable_echo_votes_verified) - { - return Ok(Vec::new()); - } - let candidate = self.candidate_mut(vote.target()); - match candidate.portable_echo_votes.get(&vote.sender()) { - Some(existing) if existing != &vote => return Err(RbcError::InvalidPortableEchoVote), - Some(_) => return Ok(Vec::new()), - None => { - candidate.portable_echo_votes.insert(vote.sender(), vote); - } - } - self.verify_portable_echo_vote_batch(vote.target())?; - Ok(self.drive(vote.target())) - } - - fn verify_portable_echo_vote_batch(&mut self, target: BlockReference) -> Result<(), RbcError> { - let votes: Vec<_> = self - .candidate(&target) - .into_iter() - .flat_map(|candidate| candidate.portable_echo_votes.values().copied()) - .collect(); - let stake: Stake = votes - .iter() - .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) - .sum(); - if !self.committee.is_quorum(stake) { - return Ok(()); - } - let digest = self.portable_echo_signature_digest(target)?; - let batch_valid = |votes: &[StarfishRbcEchoVoteV3]| { - let mut signers = AuthoritySet::default(); - for vote in votes { - signers.insert(vote.sender()); - } - let signatures: Vec<_> = votes.iter().map(|vote| vote.signature()).collect(); - let signature_refs: Vec<_> = signatures.iter().collect(); - let Some(aggregate) = bls_try_aggregate(&signature_refs) else { - return false; - }; - let Some(public_keys) = bls_public_keys_for_signers(&self.committee, signers) else { - return false; - }; - bls_fast_aggregate_verify(&digest, &aggregate, &public_keys) - }; - let valid_votes = if batch_valid(&votes) { - votes - } else { - votes - .into_iter() - .filter(|vote| { - self.committee - .get_bls_public_key(vote.sender()) - .is_some_and(|public_key| { - public_key - .verify_trusted(&digest, &vote.signature()) - .is_ok() - }) - }) - .collect() - }; - let valid_stake: Stake = valid_votes - .iter() - .map(|vote| self.committee.get_stake(vote.sender()).unwrap_or_default()) - .sum(); - let committee = Arc::clone(&self.committee); - let slot = self.slot_mut(target); - if committee.is_quorum(valid_stake) { - for vote in &valid_votes { - slot.record_phase_sender(RbcPhase::Echo, vote.sender(), target); - } - } - let candidate = slot - .candidates - .entry(target) - .or_insert_with(CandidateState::new); - candidate.portable_echo_votes = valid_votes - .iter() - .map(|vote| (vote.sender(), *vote)) - .collect(); - if committee.is_quorum(valid_stake) { - candidate.portable_echo_votes_verified = true; - for vote in valid_votes { - candidate.echoes.add(vote.sender(), &committee); - } - } - Ok(()) - } - - pub(crate) fn handle_portable_echo_qc( - &mut self, - qc: &StarfishRbcEchoQcV3, - ) -> Result, RbcError> { - if !self.echo_qc_fast_path { - return Err(RbcError::InvalidPortableEchoQc); - } - self.validate_block_ref(&qc.target())?; - if self - .candidate(&qc.target()) - .is_some_and(|candidate| candidate.portable_echo_qc_observed) - { - return Ok(self.drive(qc.target())); - } - let mut stake = 0; - for sender in qc.signers().present() { - stake += self - .committee - .get_stake(sender) - .ok_or(RbcError::InvalidPortableEchoQc)?; - } - if !self.committee.is_quorum(stake) { - return Err(RbcError::InvalidPortableEchoQc); - } - let public_keys = bls_public_keys_for_signers(&self.committee, qc.signers()) - .ok_or(RbcError::InvalidPortableEchoQc)?; - if !bls_fast_aggregate_verify( - &self.portable_echo_signature_digest(qc.target())?, - &qc.signature(), - &public_keys, - ) { - return Err(RbcError::InvalidPortableEchoQc); - } - let candidate = self.candidate_mut(qc.target()); - candidate.portable_echo_qc_observed = true; - let relay = if candidate.portable_echo_qc_emitted { - None - } else { - candidate.portable_echo_qc_emitted = true; - Some(RbcEffect::PortableEchoQc(qc.clone())) - }; - let mut effects = relay.into_iter().collect::>(); - effects.extend(self.drive(qc.target())); - if self - .candidate(&qc.target()) - .is_some_and(|candidate| candidate.header.is_none()) - { - effects.push(RbcEffect::NeedHeader { - block_ref: qc.target(), - holders: qc.signers(), - }); - } - Ok(effects) - } - /// Materialize one recipient-specific message for an untagged multicast /// effect. The network adapter calls this once per non-local recipient. pub(crate) fn make_phase_message( @@ -2241,7 +2030,6 @@ impl StarfishRbcKernel { } fn drive(&mut self, block_ref: BlockReference) -> Vec { - let committee = Arc::clone(&self.committee); let validity_threshold = self.committee.validity_threshold(); let quorum_threshold = self.committee.quorum_threshold(); let echo_qc_fast_path = self.echo_qc_fast_path; @@ -2260,37 +2048,19 @@ impl StarfishRbcKernel { let ready_trigger = candidate.echo_quorum_observed || candidate.ready_validity_observed; - let portable_vote_stake = candidate - .portable_echo_votes - .keys() - .map(|sender| committee.get_stake(*sender).unwrap_or_default()) - .sum::(); - let portable_qc_ready = echo_qc_fast_path - && !candidate.portable_echo_qc_emitted - && candidate.portable_echo_votes_verified - && committee.is_quorum(portable_vote_stake); let blocked_on_header = candidate.header.is_none() && ((can_send_ready && ready_trigger) - || (can_deliver - && (candidate.ready_quorum_observed - || candidate.portable_echo_qc_observed))); + || (can_deliver && candidate.ready_quorum_observed)); let holders = candidate.holders(); - if portable_qc_ready { - ProgressAction::EmitPortableEchoQc( - candidate.portable_echo_votes.values().copied().collect(), - ) - } else if blocked_on_header && holders != candidate.header_request_holders { + if blocked_on_header && holders != candidate.header_request_holders { candidate.header_request_holders = holders; ProgressAction::NeedHeader(holders) - } else if candidate.header.is_some() - && can_send_ready - && ready_trigger - && !(echo_qc_fast_path && candidate.portable_echo_qc_emitted) - { + } else if candidate.header.is_some() && can_send_ready && ready_trigger { ProgressAction::SendReady } else if candidate.header.is_some() && can_deliver - && (candidate.ready_quorum_observed || candidate.portable_echo_qc_observed) + && (candidate.ready_quorum_observed + || (echo_qc_fast_path && candidate.echo_quorum_observed)) { ProgressAction::Deliver } else { @@ -2335,29 +2105,6 @@ impl StarfishRbcKernel { effects.push(RbcEffect::Deliver(header)); } } - ProgressAction::EmitPortableEchoQc(votes) => { - let candidate = self.candidate_mut(block_ref); - if !candidate.portable_echo_qc_emitted { - candidate.portable_echo_qc_emitted = true; - // The service emits this QC before the following - // delivery effect. Its ordered Core bridge awaits - // creation/dissemination of the QC-bearing block, so - // delivery cannot overtake portable publication. - candidate.portable_echo_qc_observed = true; - let mut signers = AuthoritySet::default(); - for vote in &votes { - signers.insert(vote.sender()); - } - let signatures: Vec<_> = - votes.iter().map(|vote| vote.signature()).collect(); - let signature_refs: Vec<_> = signatures.iter().collect(); - effects.push(RbcEffect::PortableEchoQc(StarfishRbcEchoQcV3::new( - block_ref, - signers, - bls_aggregate(&signature_refs), - ))); - } - } ProgressAction::None => break, } } @@ -2467,8 +2214,7 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, - mac_keyrings_for_test, + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, types::{BlockDigest, BlockReference}, }; @@ -2519,7 +2265,7 @@ mod tests { #[test] fn embedded_block_references_drive_rbc_without_phase_messages() { - let committee = Committee::new_test(vec![1; 4]); + let committee = Committee::new_for_benchmarks(4); let keyrings = mac_keyrings_for_test(committee.len()); let mut receiver = StarfishRbcKernel::new( committee.clone(), @@ -2572,8 +2318,8 @@ mod tests { } #[test] - fn portable_echo_qc_fast_path_requires_and_accepts_exact_signed_quorum() { - let committee = Committee::new_test(vec![1; 4]); + fn flagged_echo_qc_fast_path_delivers_unique_header_without_ready_quorum() { + let committee = Committee::new_for_benchmarks(4); let keyrings = mac_keyrings_for_test(committee.len()); let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( committee, @@ -2591,136 +2337,26 @@ mod tests { .unwrap(); receiver.authorize_echo(target).unwrap(); - let digest = receiver.portable_echo_signature_digest(target).unwrap(); - let signer = dummy_bls_signer(); - let votes: Vec<_> = (0..3) - .map(|sender| StarfishRbcEchoVoteV3::new(target, sender, signer.sign_digest(&digest))) - .collect(); - for vote in &votes[..2] { - assert!( - !receiver - .handle_portable_echo_vote(*vote) - .unwrap() - .iter() - .any(|effect| matches!(effect, RbcEffect::Deliver(_))) - ); - } - let effects = receiver.handle_portable_echo_vote(votes[2]).unwrap(); + let echo = StarfishRbcReferenceV3::new(StarfishRbcReferenceKindV3::Echo, target); + assert!( + receiver + .handle_embedded_reference(1, echo) + .unwrap() + .is_empty() + ); + let effects = receiver.handle_embedded_reference(2, echo).unwrap(); - assert!(!effects.iter().any(|effect| matches!( + assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::MulticastPhase { phase: RbcPhase::Ready, - .. - } - ))); - let qc = effects - .iter() - .find_map(|effect| match effect { - RbcEffect::PortableEchoQc(qc) => Some(qc.clone()), - _ => None, - }) - .expect("signed quorum must emit a portable ECHO-QC"); - let qc_position = effects - .iter() - .position(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) - .unwrap(); - let delivery_position = effects - .iter() - .position(|effect| { - matches!( - effect, - RbcEffect::Deliver(header) if header.reference() == target - ) - }) - .expect("delivery follows portable publication"); - assert!(qc_position < delivery_position); - assert!(receiver.handle_portable_echo_qc(&qc).unwrap().is_empty()); - - let committee = Committee::new_test(vec![1; 4]); - let keyrings = mac_keyrings_for_test(committee.len()); - let mut late_receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( - committee, - 3, - instance(TEST_INSTANCE_BYTE), - BlockAuthenticationScheme::MacVector, - Arc::new(keyrings[3].clone()), - 1, - true, - ) - .unwrap(); - let missing = late_receiver.handle_portable_echo_qc(&qc).unwrap(); - assert!(missing.iter().any(|effect| matches!( - effect, - RbcEffect::PortableEchoQc(relayed) if relayed == &qc - ))); - assert!(missing.iter().any(|effect| matches!( - effect, - RbcEffect::NeedHeader { block_ref, .. } if *block_ref == target + block_ref, + } if *block_ref == target ))); - let effects = late_receiver - .note_header_available(pinned_header_for_context(late_receiver.context, target)) - .unwrap(); assert!(effects.iter().any(|effect| matches!( effect, RbcEffect::Deliver(header) if header.reference() == target ))); - - let other_target = block(3, 1, 0x73); - let forged = StarfishRbcEchoQcV3::new(other_target, qc.signers(), qc.signature()); - assert_eq!( - receiver.handle_portable_echo_qc(&forged), - Err(RbcError::InvalidPortableEchoQc) - ); - } - - #[test] - fn invalid_echo_vote_cannot_poison_a_later_portable_quorum() { - let committee = Committee::new_test(vec![1; 4]); - let keyrings = mac_keyrings_for_test(committee.len()); - let mut receiver = StarfishRbcKernel::new_with_echo_qc_fast_path( - committee, - 0, - instance(TEST_INSTANCE_BYTE), - BlockAuthenticationScheme::MacVector, - Arc::new(keyrings[0].clone()), - 1, - true, - ) - .unwrap(); - let target = block(3, 1, 0x74); - receiver - .note_header_available(pinned_header_for_context(receiver.context, target)) - .unwrap(); - let digest = receiver.portable_echo_signature_digest(target).unwrap(); - let signer = dummy_bls_signer(); - let wrong_signature = signer.sign_digest(&[0xFF; 32]); - for (sender, signature) in [ - (0, signer.sign_digest(&digest)), - (1, wrong_signature), - (2, signer.sign_digest(&digest)), - ] { - let effects = receiver - .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new(target, sender, signature)) - .unwrap(); - assert!( - !effects - .iter() - .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) - ); - } - let effects = receiver - .handle_portable_echo_vote(StarfishRbcEchoVoteV3::new( - target, - 3, - signer.sign_digest(&digest), - )) - .unwrap(); - assert!( - effects - .iter() - .any(|effect| matches!(effect, RbcEffect::PortableEchoQc(_))) - ); } fn valid_canonical_header( @@ -2911,9 +2547,6 @@ mod tests { ); deliveries[owner as usize].push(header.reference()); } - RbcEffect::PortableEchoVote(_) | RbcEffect::PortableEchoQc(_) => { - panic!("portable ECHO effects require the single-DAG test harness") - } } } (deliveries, recoveries) @@ -3061,36 +2694,6 @@ mod tests { } } - #[test] - fn single_dag_digest_binds_portable_echo_votes_and_certificate() { - let target = block(0, 4, 0x43); - let signature = dummy_bls_signer().sign_digest(&[0x44; 32]); - let vote = StarfishRbcEchoVoteV3::new(target, 1, signature); - let mut signers = AuthoritySet::default(); - signers.insert(0); - signers.insert(1); - signers.insert(2); - let signatures = [&signature, &signature, &signature]; - let qc = StarfishRbcEchoQcV3::new(target, signers, bls_aggregate(&signatures)); - let empty = StarfishRbcFieldsV3::default(); - let with_vote = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), vec![vote], Vec::new()); - let with_qc = StarfishRbcFieldsV3::with_portable_echo(Vec::new(), Vec::new(), vec![qc]); - let digest = |fields: &StarfishRbcFieldsV3| { - BlockDigest::new_starfish_rbc_single_dag_header( - 3, - 5, - &[], - &[], - 7, - TransactionsCommitment::default(), - fields, - ) - }; - assert_ne!(digest(&empty), digest(&with_vote)); - assert_ne!(digest(&empty), digest(&with_qc)); - assert_ne!(digest(&with_vote), digest(&with_qc)); - } - #[test] fn canonical_header_validation_is_authentication_independent_and_pins_content() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 90941860..8b08038a 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -25,7 +25,7 @@ use tokio::{ use crate::{ committee::Committee, - crypto::{BlsSigner, MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, + crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, network::NetworkMessage, starfish_rbc::{ PinnedRbcHeader, RbcCanonicalHeader, RbcEffect, RbcError, RbcHeaderProposal, @@ -34,8 +34,8 @@ use crate::{ }, types::{ AuthorityIndex, AuthoritySet, BlockAuthenticationScheme, BlockDigest, BlockReference, - RoundNumber, StarfishRbcEchoQcV3, StarfishRbcEchoVoteV3, StarfishRbcFieldsV3, - StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, TimestampNs, TransactionData, + RoundNumber, StarfishRbcFieldsV3, StarfishRbcReferenceKindV3, StarfishRbcReferenceV3, + TimestampNs, TransactionData, }, }; @@ -48,7 +48,7 @@ pub(crate) enum RbcInitialAuthenticator { Ed25519(Signer), MlDsa44(MlDsa44Signer), MlDsa65(MlDsa65Signer), - Mac(Signer, BlsSigner), + Mac, } impl RbcInitialAuthenticator { @@ -57,7 +57,7 @@ impl RbcInitialAuthenticator { Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, - Self::Mac(_, _) => BlockAuthenticationScheme::MacVector, + Self::Mac => BlockAuthenticationScheme::MacVector, } } } @@ -160,8 +160,6 @@ pub(crate) enum RbcServiceEvent { /// An irrevocable local phase statement waiting to be embedded in the /// next ordinary Starfish block. ReferenceReady(StarfishRbcReferenceV3), - EchoVoteReady(StarfishRbcEchoVoteV3), - EchoQcReady(StarfishRbcEchoQcV3), Rejected { peer: Option, error: RbcServiceError, @@ -437,14 +435,7 @@ fn validate_local_authenticator( RbcInitialAuthenticator::MlDsa65(signer) => committee .get_ml_dsa_65_public_key(own_authority) .is_some_and(|public_key| public_key == &signer.public_key()), - RbcInitialAuthenticator::Mac(signer, echo_signer) => { - committee - .get_public_key(own_authority) - .is_some_and(|public_key| public_key == &signer.public_key()) - && committee - .get_bls_public_key(own_authority) - .is_some_and(|public_key| public_key == &echo_signer.public_key()) - } + RbcInitialAuthenticator::Mac => committee.known_authority(own_authority), }; if matches { Ok(()) @@ -617,7 +608,7 @@ impl RbcServiceState { RbcInitialProof::MlDsa65(signer.sign_digest(&BlockDigest::from(digest))); self.public_initial_proposals(header, proof, transaction_data) } - RbcInitialAuthenticator::Mac(_, _) => self + RbcInitialAuthenticator::Mac => self .committee .authorities() .filter(|recipient| *recipient != self.own_authority) @@ -826,27 +817,6 @@ impl RbcServiceState { self.pending_fetches.remove(&header.reference()); let _ = self.events.send(RbcServiceEvent::Delivered(header)); } - RbcEffect::PortableEchoVote(target) => { - let RbcInitialAuthenticator::Mac(_, signer) = &self.initial_authenticator - else { - self.reject(None, RbcError::InvalidPortableEchoVote.into()); - continue; - }; - match self.kernel.portable_echo_signature_digest(target) { - Ok(digest) => { - let vote = StarfishRbcEchoVoteV3::new( - target, - self.own_authority, - signer.sign_digest(&digest), - ); - let _ = self.events.send(RbcServiceEvent::EchoVoteReady(vote)); - } - Err(error) => self.reject(None, error.into()), - } - } - RbcEffect::PortableEchoQc(qc) => { - let _ = self.events.send(RbcServiceEvent::EchoQcReady(qc)); - } } } } @@ -875,18 +845,6 @@ impl RbcServiceState { Err(error) => self.reject(Some(sender), error.into()), } } - for vote in references.echo_votes() { - match self.kernel.handle_portable_echo_vote(*vote) { - Ok(effects) => self.process_effects(effects), - Err(error) => self.reject(Some(sender), error.into()), - } - } - for qc in references.echo_qcs() { - match self.kernel.handle_portable_echo_qc(qc) { - Ok(effects) => self.process_effects(effects), - Err(error) => self.reject(Some(sender), error.into()), - } - } } fn note_pending_fetch(&mut self, block_ref: BlockReference, holders: AuthoritySet) { @@ -1022,8 +980,7 @@ mod tests { use super::*; use crate::{ crypto::{ - dummy_bls_signer, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, - mac_keyrings_for_test, + dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, starfish_rbc::RbcPhase, types::{TransactionData, VerifiedBlock}, @@ -1058,9 +1015,7 @@ mod tests { let keyrings = mac_keyrings_for_test(4); let authenticator = match scheme { BlockAuthenticationScheme::Ed25519 => RbcInitialAuthenticator::Ed25519(dummy_signer()), - BlockAuthenticationScheme::MacVector => { - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()) - } + BlockAuthenticationScheme::MacVector => RbcInitialAuthenticator::Mac, BlockAuthenticationScheme::MlDsa44 => { RbcInitialAuthenticator::MlDsa44(dummy_ml_dsa_44_signer()) } @@ -1094,7 +1049,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedCarrierDag, @@ -1115,7 +1070,7 @@ mod tests { instance(), BlockAuthenticationScheme::MacVector, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(3_600), RbcPhaseAuthorityV1::EmbeddedSingleDag { @@ -1125,29 +1080,6 @@ mod tests { .unwrap() } - fn start_single_dag_fast_service() -> ( - RbcServiceHandle, - mpsc::UnboundedReceiver, - JoinHandle<()>, - ) { - let committee = Committee::new_test(vec![1; 4]); - let keyrings = mac_keyrings_for_test(4); - start_starfish_rbc_service_with_phase_authority( - committee, - 0, - instance(), - BlockAuthenticationScheme::MacVector, - Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), - 1, - Duration::from_secs(3_600), - RbcPhaseAuthorityV1::EmbeddedSingleDag { - echo_qc_fast_path: true, - }, - ) - .unwrap() - } - async fn next_event(events: &mut mpsc::UnboundedReceiver) -> RbcServiceEvent { tokio::time::timeout(Duration::from_secs(2), events.recv()) .await @@ -1291,44 +1223,6 @@ mod tests { task.await.unwrap(); } - #[tokio::test] - async fn portable_fast_path_emits_signed_echo_vote_without_phase_message() { - let (handle, mut events, task) = start_single_dag_fast_service(); - let mut header = local_header(1, 4); - header.starfish_rbc_v3 = Some(StarfishRbcFieldsV3::default()); - let canonical = handle.start_local_header(header).await.unwrap(); - let mut initials = 0; - let mut vote = None; - for _ in 0..5 { - match next_event(&mut events).await { - RbcServiceEvent::HeaderStaged(header) => { - assert_eq!(header.reference(), canonical.reference()); - } - RbcServiceEvent::Network { - message: NetworkMessage::RbcInitial(_), - .. - } => initials += 1, - RbcServiceEvent::EchoVoteReady(echo_vote) => vote = Some(echo_vote), - RbcServiceEvent::ReferenceReady(reference) - if reference.kind() == StarfishRbcReferenceKindV3::Echo => - { - panic!("portable mode emitted an unsigned ECHO reference") - } - RbcServiceEvent::Network { - message: NetworkMessage::RbcPhase(_), - .. - } => panic!("portable mode emitted a standalone phase message"), - event => panic!("unexpected portable startup event: {event:?}"), - } - } - assert_eq!(initials, 3); - let vote = vote.expect("portable mode must emit one signed ECHO vote"); - assert_eq!(vote.target(), canonical.reference()); - assert_eq!(vote.sender(), 0); - drop(handle); - task.await.unwrap(); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blocking_local_start_waits_for_kernel_selection_and_event_enqueue() { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); @@ -1726,7 +1620,7 @@ mod tests { instance(), BlockAuthenticationScheme::Ed25519, Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac(dummy_signer(), dummy_bls_signer()), + RbcInitialAuthenticator::Mac, 1, Duration::from_secs(1), ); diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index d5fd3f90..ae1f9523 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -410,16 +410,6 @@ impl Syncer { self.try_new_block(BlockCreationReason::CertificateEvent); } - pub fn apply_starfish_rbc_echo_vote(&mut self, vote: crate::types::StarfishRbcEchoVoteV3) { - self.core.add_starfish_rbc_echo_vote(vote); - self.try_new_block(BlockCreationReason::CertificateEvent); - } - - pub fn apply_starfish_rbc_echo_qc(&mut self, qc: crate::types::StarfishRbcEchoQcV3) { - self.core.add_starfish_rbc_echo_qc(qc); - self.try_new_block(BlockCreationReason::CertificateEvent); - } - /// Sequence one exact deterministic carrier-frontier delta. In M7 this is /// the sole application-ordering authority; the legacy Starfish committer /// remains disabled in this mode. diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 2afdb77a..6edca2ff 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -138,78 +138,6 @@ pub struct StarfishRbcReferenceV3 { reference: BlockReference, } -/// A publicly verifiable ECHO vote carried by an ordinary single-DAG block. -/// -/// The signature is deliberately independent of the carrying block. A later -/// block can therefore copy a quorum of votes into a portable certificate -/// without introducing a standalone RBC phase message. -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] -pub struct StarfishRbcEchoVoteV3 { - target: BlockReference, - sender: AuthorityIndex, - signature: BlsSignatureBytes, -} - -impl StarfishRbcEchoVoteV3 { - pub fn new( - target: BlockReference, - sender: AuthorityIndex, - signature: BlsSignatureBytes, - ) -> Self { - Self { - target, - sender, - signature, - } - } - - pub fn target(self) -> BlockReference { - self.target - } - - pub fn sender(self) -> AuthorityIndex { - self.sender - } - - pub fn signature(self) -> BlsSignatureBytes { - self.signature - } -} - -/// Portable quorum certificate over exact, publicly verifiable ECHO votes. -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] -pub struct StarfishRbcEchoQcV3 { - target: BlockReference, - signers: AuthoritySet, - signature: BlsSignatureBytes, -} - -impl StarfishRbcEchoQcV3 { - pub fn new( - target: BlockReference, - signers: AuthoritySet, - signature: BlsSignatureBytes, - ) -> Self { - Self { - target, - signers, - signature, - } - } - - pub fn target(&self) -> BlockReference { - self.target - } - - pub fn signers(&self) -> AuthoritySet { - self.signers - } - - pub fn signature(&self) -> BlsSignatureBytes { - self.signature - } -} - impl StarfishRbcReferenceV3 { pub fn new(kind: StarfishRbcReferenceKindV3, reference: BlockReference) -> Self { Self { kind, reference } @@ -232,53 +160,19 @@ impl StarfishRbcReferenceV3 { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct StarfishRbcFieldsV3 { references: Vec, - #[serde(default)] - echo_votes: Vec, - #[serde(default)] - echo_qcs: Vec, } impl StarfishRbcFieldsV3 { pub fn new(mut references: Vec) -> Self { references.sort_unstable(); references.dedup(); - Self { - references, - echo_votes: Vec::new(), - echo_qcs: Vec::new(), - } - } - - pub fn with_portable_echo( - mut references: Vec, - mut echo_votes: Vec, - mut echo_qcs: Vec, - ) -> Self { - references.sort_unstable(); - references.dedup(); - echo_votes.sort_unstable(); - echo_votes.dedup(); - echo_qcs.sort_unstable(); - echo_qcs.dedup(); - Self { - references, - echo_votes, - echo_qcs, - } + Self { references } } pub fn references(&self) -> &[StarfishRbcReferenceV3] { &self.references } - pub fn echo_votes(&self) -> &[StarfishRbcEchoVoteV3] { - &self.echo_votes - } - - pub fn echo_qcs(&self) -> &[StarfishRbcEchoQcV3] { - &self.echo_qcs - } - pub(crate) fn validate_for_block( &self, committee: &Committee, @@ -287,54 +181,17 @@ impl StarfishRbcFieldsV3 { if self.references.len() > committee.len().saturating_mul(6) { return false; } - if self.echo_votes.len() > committee.len().saturating_mul(3) - || self.echo_qcs.len() > committee.len() - { - return false; - } if self.references.windows(2).any(|pair| pair[0] >= pair[1]) { return false; } let mut statements = AHashSet::new(); - let references_valid = self.references.iter().all(|evidence| { + self.references.iter().all(|evidence| { let reference = evidence.reference(); reference.round > 0 && reference.round <= block_round && committee.known_authority(reference.authority) && statements.insert((evidence.kind(), reference.authority, reference.round)) - }); - if !references_valid - || self.echo_votes.windows(2).any(|pair| pair[0] >= pair[1]) - || self.echo_qcs.windows(2).any(|pair| pair[0] >= pair[1]) - { - return false; - } - let votes_valid = self.echo_votes.iter().all(|vote| { - let target = vote.target(); - target.round > 0 - && target.round <= block_round - && committee.known_authority(target.authority) - && committee.known_authority(vote.sender()) - }); - votes_valid - && self.echo_qcs.iter().all(|qc| { - let target = qc.target(); - if target.round == 0 - || target.round >= block_round - || !committee.known_authority(target.authority) - || qc.signers().is_empty() - { - return false; - } - let mut stake = 0; - for sender in qc.signers().present() { - if !committee.known_authority(sender) { - return false; - } - stake += committee.get_stake(sender).unwrap_or_default(); - } - committee.is_quorum(stake) - }) + }) } } @@ -2502,9 +2359,7 @@ fn verify_signed_quorum( Ok(()) } -#[derive( - Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Serialize, Deserialize, Default, Debug, -)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, Default, Debug)] pub struct AuthoritySet([u64; MAX_COMMITTEE_WORDS]); pub type TimestampNs = u64; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 8a8f14ed..01847d74 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -210,8 +210,9 @@ enum Operation { /// Requires embedded RBC-DAG authority and changes the finality proof. #[clap(long, default_value_t = false)] starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only: deliver a single-DAG RBC header from a portable - /// quorum of publicly signed ECHO votes carried in ordinary blocks. + /// Testbed-only: deliver a single-DAG RBC header after quorum ECHO. + /// This preserves uniqueness but not Byzantine selective-withholding + /// totality, so it is restricted to finite benchmark runs. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. @@ -615,7 +616,7 @@ async fn local_benchmark( } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( - "Single-DAG portable ECHO-QC delivery: ENABLED (signed votes; testbed evaluation)" + "Single-DAG ECHO-QC delivery: ENABLED (testbed-only; Byzantine totality not claimed)" ); } if let Some(latency) = node_parameters.uniform_latency_ms { diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index 233a637b..de2e058d 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -35,10 +35,8 @@ is one of: - `Ready(BlockReference)`. The carrying block's authenticated author is the statement sender. The list is -part of that ordinary block's content digest. Consequently the default path -needs no standalone phase MAC, signature, or phase network message. The -portable fast path adds embedded BLS ECHO votes and an aggregate QC, but still -adds no phase message. +part of that ordinary block's content digest. Consequently no standalone phase +MAC, signature, or phase network message is needed. References are sorted, duplicate-free and bounded by `6 * committee_size` per block. A sender may name at most one digest for each `(phase, target author, @@ -89,29 +87,19 @@ create a second physical round counter. - Recovery content must recompute to the exact requested `BlockReference`. - Frozen direct-RBC and carrier-DAG formats retain their old domains. -### Portable ECHO-QC fast path +### Explicit testbed ECHO-QC fast path Finite benchmarks may opt into -`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode ECHO is a compact -BLS vote over the exact target reference, protocol instance and committee. -Votes ride in ordinary DAG blocks. Once quorum stake verifies, the node batch -aggregates them into one portable certificate containing the target, signer -bitmap and 48-byte aggregate signature. The QC also rides in an ordinary DAG -block; there is no standalone ECHO or READY message. - -An honest node publishes or relays the QC before its delivery effect. The -ordered Core bridge queues the QC-bearing block before applying delivery. -Quorum intersection gives uniqueness, while public verification and mandatory -relay remove the old receiver-local selective-withholding caveat: any valid QC -that reaches one honest validator can be verified and propagated by every -other validator. Missing target content still uses exact header recovery and -delivery remains fail-closed until that content is present. Invalid aggregate -batches fall back to individual vote verification so one Byzantine vote cannot -poison an otherwise valid quorum. - -The flag remains testbed-only because bounded retirement, durable pending-QC -replay and a complete asynchronous proof are still production follow-ups. The -default V3 path continues to use the conventional quorum-READY rule. +`--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode a node delivers an +exact header as soon as it has quorum locked ECHO statements, while still +emitting the normal READY statement. Quorum intersection prevents two +different headers from both obtaining such a certificate, so agreement is +preserved. The current wire format does not carry the exact ECHO witness set, +however: Byzantine ECHO senders can selectively reveal their statements so +one honest node obtains quorum while another never can. Consequently the flag +does **not** provide the normal RBC totality guarantee and is rejected for +unbounded/production runs. The default V3 path continues to require quorum +READY. ## Required validation @@ -131,13 +119,10 @@ with MAC authentication and the fixed 50 ms V3 round limiter: | --- | ---: | ---: | ---: | ---: | | `starfish-rbc-single-dag` | zero | 498.3 ms | 1,000 | 0.79 MB/s | | `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | -| V3 + old receiver-local ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | -| V3 + old receiver-local ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | -| V3 + portable aggregate ECHO-QC | zero | 405.8 ms | 1,000 | 1.06 MB/s | -| V3 + portable aggregate ECHO-QC | AWS table | 983.7 ms | 1,000 | 0.74 MB/s | +| V3 + flagged ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | +| V3 + flagged ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | | `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | All exact offered transactions committed during the bounded drain. These are -single-machine research measurements, not production claims. The two old -receiver-local rows preserve the historical totality caveat; the new portable -rows use the signed aggregate certificate described above. +single-machine research measurements, not production claims. In particular, +both ECHO-QC rows carry the totality limitation above. From 4971889ac15642b4e8569635770dacf01f51be92 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:27:48 +0200 Subject: [PATCH 50/62] keep receiver-local single-DAG lower bound --- crates/starfish-core/src/config.rs | 9 ++-- .../starfish-core/src/starfish_rbc_service.rs | 13 ++++++ crates/starfish/src/main.rs | 9 ++-- docs/starfish-rbc-single-dag-v3.md | 46 +++++++++++++------ 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index fad2d43e..6babc7eb 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -94,10 +94,11 @@ pub struct NodeParameters { /// Starfish finality proof shape. #[serde(default)] pub starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only single-DAG RBC path: deliver an exact header after quorum - /// ECHO rather than quorum READY. Quorum intersection preserves a unique - /// value, but selective Byzantine ECHO withholding can violate totality; - /// this must remain an explicit benchmark flag. + /// Testbed-only receiver-local single-DAG RBC path: deliver an exact header + /// after locally observing quorum ECHO rather than quorum READY. Quorum + /// intersection preserves a unique value, but pairwise-MAC testimony is + /// not transferable and selective Byzantine withholding can violate + /// totality. This must remain an explicit benchmark flag. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Benchmark-only profile that writes the framed shadow WAL in order but diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 8b08038a..5165b3b8 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -706,6 +706,16 @@ impl RbcServiceState { return; } let Some(fetch) = self.pending_fetches.get(&block_ref) else { + // A quorum may answer the same content request concurrently. Once + // one response has pinned the exact header, later identical + // responses are benign rather than protocol rejections. + if self + .kernel + .pinned_header(block_ref) + .is_ok_and(|header| header.is_some()) + { + return; + } self.reject( Some(peer), RbcServiceError::UnexpectedHeaderResponse(block_ref), @@ -1599,6 +1609,9 @@ mod tests { )); } + // Another holder may have answered the same request concurrently. + // Once the exact header is pinned, that duplicate is idempotent. + handle.header_response(1, canonical).unwrap(); handle.retry_headers().await.unwrap(); assert!( tokio::time::timeout(Duration::from_millis(20), events.recv()) diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 01847d74..0b3f1481 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -210,9 +210,10 @@ enum Operation { /// Requires embedded RBC-DAG authority and changes the finality proof. #[clap(long, default_value_t = false)] starfish_rbc_dag_vote_qc_fast_path: bool, - /// Testbed-only: deliver a single-DAG RBC header after quorum ECHO. - /// This preserves uniqueness but not Byzantine selective-withholding - /// totality, so it is restricted to finite benchmark runs. + /// Testbed-only: deliver a single-DAG RBC header after a receiver-local + /// quorum ECHO. This preserves uniqueness but not Byzantine + /// selective-withholding totality, so it is restricted to finite + /// benchmark runs. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, /// Override only the autonomous RBC-DAG logical C2 fallback timeout. @@ -616,7 +617,7 @@ async fn local_benchmark( } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( - "Single-DAG ECHO-QC delivery: ENABLED (testbed-only; Byzantine totality not claimed)" + "Single-DAG receiver-local quorum-ECHO: ENABLED (signature-free latency lower bound; Byzantine totality not provided)" ); } if let Some(latency) = node_parameters.uniform_latency_ms { diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index de2e058d..9fb03b4f 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -87,19 +87,32 @@ create a second physical round counter. - Recovery content must recompute to the exact requested `BlockReference`. - Frozen direct-RBC and carrier-DAG formats retain their old domains. -### Explicit testbed ECHO-QC fast path +### Receiver-local quorum-ECHO latency lower bound Finite benchmarks may opt into `--starfish-rbc-single-dag-echo-qc-fast-path`. In that mode a node delivers an exact header as soon as it has quorum locked ECHO statements, while still -emitting the normal READY statement. Quorum intersection prevents two -different headers from both obtaining such a certificate, so agreement is -preserved. The current wire format does not carry the exact ECHO witness set, -however: Byzantine ECHO senders can selectively reveal their statements so -one honest node obtains quorum while another never can. Consequently the flag -does **not** provide the normal RBC totality guarantee and is rejected for -unbounded/production runs. The default V3 path continues to require quorum -READY. +emitting the normal READY statement. There is no portable QC object, witness +vector, public signature, standalone phase message, or additional fast-path +communication: each receiver acts only on ordinary DAG blocks that it +authenticated directly with its recipient-specific MAC. + +Quorum intersection and the per-sender slot locks prevent two conflicting +headers from both obtaining honest receiver-local quorums, so the experiment +retains integrity and agreement/uniqueness. It does **not** provide normal RBC +totality: Byzantine ECHO senders can selectively reveal their statements so +one honest node obtains quorum and delivers while another honest node never +can. Relaying header bytes does not transfer pairwise-MAC testimony. The flag +is therefore a finite-testbed latency lower bound, is rejected for unbounded +or production runs, and must not be described as a Byzantine-totality-safe +RBC. The default V3 path continues to require quorum READY. + +Two attempted ways to make this fast-path evidence portable were removed. A +public aggregate-signature certificate violated the signature-free design +constraint. A complete per-recipient MAC vector plus exact witness references +remained signature-free, but made proposals and certificates grow with the +committee and overloaded the n=40 single-machine testbed. Neither proof format +is part of the current V3 wire protocol. ## Required validation @@ -110,19 +123,22 @@ withholding with bounded recovery, restart replay and deterministic Starfish commit order. Matched n=10 and n=40 zero/AWS runs must compare V3 against the frozen carrier baseline using identical load and duration. -## Initial n=10 testbed checkpoint +## Testbed checkpoint -The first matched local runs used 1,000 offered transactions/s for 20 seconds -with MAC authentication and the fixed 50 ms V3 round limiter: +Matched local runs used 1,000 offered transactions/s for 20 seconds with MAC +authentication and the fixed 50 ms V3 round limiter: | Protocol/profile | Network | p50 E2E | Eventual TPS | Outbound/node | | --- | ---: | ---: | ---: | ---: | | `starfish-rbc-single-dag` | zero | 498.3 ms | 1,000 | 0.79 MB/s | | `starfish-rbc-single-dag` | AWS table | 1,285.7 ms | 1,000 | 0.61 MB/s | -| V3 + flagged ECHO-QC | AWS table | 961.0 ms | 1,000 | 0.61 MB/s | -| V3 + flagged ECHO-QC, n=40 | AWS table | 977.35 ms | 1,000 | 3.04 MB/s | +| V3 + receiver-local quorum-ECHO, n=10 | zero | 400.5 ms | 1,000 | 0.79 MB/s | +| V3 + receiver-local quorum-ECHO, n=10 | AWS table | 956.0 ms | 1,000 | 0.61 MB/s | +| V3 + receiver-local quorum-ECHO, n=40 | zero | 390.2 ms | 1,000 | 5.88 MB/s | +| V3 + receiver-local quorum-ECHO, n=40 | AWS table | 968.2 ms | 1,000 | 3.07 MB/s | | `starfish-mac` lower bound | AWS table | 610.0 ms | 1,000 | 0.61 MB/s | All exact offered transactions committed during the bounded drain. These are single-machine research measurements, not production claims. In particular, -both ECHO-QC rows carry the totality limitation above. +the receiver-local rows preserve integrity and agreement but carry the +Byzantine-totality limitation above. From cdbb5fba3f60dbedb180ca2db1239eddcdbda9cd Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:37:01 +0200 Subject: [PATCH 51/62] Revert "Add flagged RBC-DAG vote-QC fast path" This reverts commit f0ec1457f99a511fc0f41b9ddfde6f4486d2d675. --- README.md | 9 - crates/starfish-core/src/config.rs | 11 - crates/starfish-core/src/net_sync.rs | 173 ++++----------- .../src/starfish_rbc_dag/model.rs | 81 ++----- .../src/starfish_rbc_dag/projection.rs | 16 +- .../src/starfish_rbc_dag_shadow.rs | 121 +---------- .../src/starfish_rbc_dag_shadow_service.rs | 197 ++++-------------- crates/starfish-core/src/validator.rs | 79 +------ crates/starfish/src/main.rs | 18 -- docs/starfish-rbc-dag-protocol.md | 15 +- 10 files changed, 110 insertions(+), 610 deletions(-) diff --git a/README.md b/README.md index d1aa5652..b39e91ba 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,6 @@ single-slot exact synchronization rather than checkpoint or proof-safe late-node Its current authoritative journal uses the V4 autonomous WAL namespace with `SRD5` raw records so older traces cannot be reinterpreted under the optimistic-delivery rules. -Strict two-level Starfish finality remains the default. The local benchmark additionally exposes -`--starfish-rbc-dag-vote-qc-fast-path`, a deliberately flagged testbed experiment that commits an -exact projected leader as soon as its projected vote quorum is present instead of waiting for the -second certifier wave. It sends no additional protocol messages, but changes the proof shape and -must not be reported as the strict or production Starfish result. - The default WAL syncs every transition. `--starfish-rbc-dag-shadow-buffered-wal` preserves ordered frames but syncs only on clean shutdown and is not crash-safe. Actor replay covers the state explicitly documented in the protocol design; full validator crash recovery, bounded checkpoint @@ -360,9 +354,6 @@ cargo run --release --bin starfish -- local-benchmark \ The buffered WAL is benchmark-only and is not crash-safe. -To measure the separately labelled testbed fast path, append -`--starfish-rbc-dag-vote-qc-fast-path`. Omitting it always measures the strict two-level rule. - ### Local dryrun with monitoring and dashboard The dryrun script launches a Docker-based local testbed with diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 6babc7eb..9e9c3749 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -88,12 +88,6 @@ pub struct NodeParameters { /// transport but its phase messages cannot mark a block clean. #[serde(default)] pub starfish_rbc_dag_embedded_rbc_authority: bool, - /// Testbed-only optimistic commit path: finalize an exact projected - /// leader after its projected vote quorum, without waiting for the second - /// certifier quorum. Disabled by default because this changes the strict - /// Starfish finality proof shape. - #[serde(default)] - pub starfish_rbc_dag_vote_qc_fast_path: bool, /// Testbed-only receiver-local single-DAG RBC path: deliver an exact header /// after locally observing quorum ECHO rather than quorum READY. Quorum /// intersection preserves a unique value, but pairwise-MAC testimony is @@ -186,7 +180,6 @@ impl Default for NodeParameters { starfish_rbc_dag_autonomous_clock: false, starfish_rbc_dag_consensus_timeout: None, starfish_rbc_dag_embedded_rbc_authority: false, - starfish_rbc_dag_vote_qc_fast_path: false, starfish_rbc_single_dag_echo_qc_fast_path: false, starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), @@ -462,7 +455,6 @@ mod tests { assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert_eq!(parameters.starfish_rbc_dag_consensus_timeout, None); - assert!(!parameters.starfish_rbc_dag_vote_qc_fast_path); assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); @@ -478,7 +470,6 @@ mod tests { assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert_eq!(decoded.starfish_rbc_dag_consensus_timeout, None); - assert!(!decoded.starfish_rbc_dag_vote_qc_fast_path); assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); } @@ -490,7 +481,6 @@ mod tests { starfish_rbc_dag_autonomous_clock: true, leader_timeout: Duration::from_millis(125), starfish_rbc_dag_consensus_timeout: Some(Duration::from_millis(75)), - starfish_rbc_dag_vote_qc_fast_path: true, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -499,7 +489,6 @@ mod tests { let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); assert!(decoded.starfish_rbc_dag_shadow); assert!(decoded.starfish_rbc_dag_autonomous_clock); - assert!(decoded.starfish_rbc_dag_vote_qc_fast_path); assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(decoded.leader_timeout, Duration::from_millis(125)); assert_eq!( diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 9f4ec5e7..f7765ad0 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -45,7 +45,7 @@ use crate::{ RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, UtilizationTimerVecExt, }, - network::{BlockBatch, Connection, Network, NetworkMessage, RbcDagShadowCarrier, ShardPayload}, + network::{BlockBatch, Connection, Network, NetworkMessage, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, sailfish_service::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, @@ -350,32 +350,21 @@ impl RbcDagOutboundMailboxV1 { } } - #[cfg(test)] fn enqueue( &self, message: NetworkMessage, - committee: &RbcDagCommitteeContextV1, - ) -> Result { - self.enqueue_with_proactive_reference(message, committee, None) - } - - fn enqueue_with_proactive_reference( - &self, - message: NetworkMessage, - committee: &RbcDagCommitteeContextV1, - proactive_reference: Option, + committee: &Committee, ) -> Result { if let Some(reason) = self.inner.state.lock().failure.clone() { return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); } - let (class, key) = - match rbc_dag_outbound_classification(&message, committee, proactive_reference) { - Ok(classification) => classification, - Err(error) => { - self.fail(&error); - return Err(error); - } - }; + let (class, key) = match rbc_dag_outbound_classification(&message, committee) { + Ok(classification) => classification, + Err(error) => { + self.fail(&error); + return Err(error); + } + }; let framed_bytes = match bincode::serialized_size(&message) .map_err(|error| RbcDagOutboundMailboxErrorV1::Serialization(error.to_string())) .and_then(|size| { @@ -463,19 +452,27 @@ impl RbcDagOutboundMailboxV1 { fn rbc_dag_outbound_classification( message: &NetworkMessage, - committee: &RbcDagCommitteeContextV1, - proactive_reference: Option, + committee: &Committee, ) -> Result<(RbcDagOutboundClassV1, RbcDagOutboundKeyV1), RbcDagOutboundMailboxErrorV1> { let priority = RbcDagOutboundClassV1::Priority; match message { NetworkMessage::RbcDagShadowCarrier(carrier) => { - let reference = match proactive_reference { - Some(reference) => reference, - None => rbc_dag_proactive_reference(carrier, committee)?, - }; + let candidate = + CandidateCarrierV1::decode_wire(&carrier.canonical_carrier, committee, None) + .map_err(|error| { + RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) + })?; + let canonical = candidate.canonical_wire_bytes().map_err(|error| { + RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) + })?; + if canonical != carrier.canonical_carrier { + return Err(RbcDagOutboundMailboxErrorV1::InvalidProactive( + "non-canonical carrier wire".to_owned(), + )); + } Ok(( RbcDagOutboundClassV1::Proactive, - RbcDagOutboundKeyV1::Proactive(reference), + RbcDagOutboundKeyV1::Proactive(candidate.reference()), )) } NetworkMessage::RbcDagShadowCarrierRequest(reference) => { @@ -509,24 +506,6 @@ fn rbc_dag_outbound_classification( } } -fn rbc_dag_proactive_reference( - carrier: &RbcDagShadowCarrier, - committee: &RbcDagCommitteeContextV1, -) -> Result { - let candidate = - CandidateCarrierV1::decode_wire_with_committee(&carrier.canonical_carrier, committee, None) - .map_err(|error| RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()))?; - let canonical = candidate - .canonical_wire_bytes() - .map_err(|error| RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()))?; - if canonical != carrier.canonical_carrier { - return Err(RbcDagOutboundMailboxErrorV1::InvalidProactive( - "non-canonical carrier wire".to_owned(), - )); - } - Ok(candidate.reference()) -} - fn rbc_dag_outbound_messages_equal(left: &NetworkMessage, right: &NetworkMessage) -> bool { match (left, right) { (NetworkMessage::RbcDagShadowCarrier(left), NetworkMessage::RbcDagShadowCarrier(right)) => { @@ -3217,10 +3196,6 @@ impl NetworkSyncer } else { (None, None, None) }; - let rbc_dag_committee_context = recovered_shadow_local_headers.as_ref().map(|_| { - RbcDagCommitteeContextV1::new(committee.clone()) - .expect("validated committee must initialize the RBC-DAG shadow") - }); let (starfish_rbc_dag_shadow_service, rbc_dag_shadow_event_rx, rbc_dag_shadow_service_task) = if let Some(recovered_local_headers) = recovered_shadow_local_headers { let protocol_instance_bytes = node_parameters @@ -3230,9 +3205,8 @@ impl NetworkSyncer protocol_instance_bytes, node_parameters.starfish_rbc_dag_autonomous_clock, ); - let committee_context = rbc_dag_committee_context - .clone() - .expect("RBC-DAG runtime must retain its validated committee context"); + let committee_context = RbcDagCommitteeContextV1::new(committee.clone()) + .expect("validated committee must initialize the RBC-DAG shadow"); let context = RbcDagContextV1::new_with_committee( protocol_instance, &committee_context, @@ -3287,7 +3261,6 @@ impl NetworkSyncer Arc::clone(&metrics), rbc_dag_frontier_recovery_cursor, !rbc_dag_clock_start_paused, - node_parameters.starfish_rbc_dag_vote_qc_fast_path, ) } else { let start = if rbc_dag_clock_start_paused { @@ -3657,20 +3630,12 @@ impl NetworkSyncer }; let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); - let rbc_dag_committee_context = rbc_dag_committee_context - .clone() - .expect("RBC-DAG event router must retain its validated committee context"); let shadow_metrics = metrics.clone(); let rbc_dag_clock_bridge_tx = rbc_dag_clock_bridge_tx.clone(); let rbc_dag_core_control_tx = rbc_dag_core_control_tx; let rbc_dag_assignment_tx = rbc_dag_assignment_tx; let rbc_dag_shutdown_started = rbc_dag_shutdown_started; handle.spawn(async move { - // A local broadcast emits the same canonical carrier once per - // recipient. Validate it on the first event and reuse only its - // exact reference while the bytes remain identical; receiver - // authentication and canonical decoding are unchanged. - let mut last_proactive_carrier: Option<(Vec, BlockReference)> = None; let mut router_guard = embedded_rbc_authority.then(|| { RbcDagEventRouterGuardV1::new( shadow_metrics.clone(), @@ -3699,49 +3664,7 @@ impl NetworkSyncer .get(&recipient) .cloned(); if let Some(mailbox) = mailbox { - let proactive_reference = match &message { - NetworkMessage::RbcDagShadowCarrier(carrier) => { - if let Some(reference) = last_proactive_carrier - .as_ref() - .filter(|(canonical, _)| { - canonical == &carrier.canonical_carrier - }) - .map(|(_, reference)| *reference) - { - Some(reference) - } else { - match rbc_dag_proactive_reference( - carrier, - &rbc_dag_committee_context, - ) { - Ok(reference) => { - last_proactive_carrier = Some(( - carrier.canonical_carrier.clone(), - reference, - )); - Some(reference) - } - Err(error) => { - mailbox.fail(&error); - fail_rbc_dag_outbound_transport( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - embedded_rbc_authority, - recipient, - &error, - ); - continue; - } - } - } - } - _ => None, - }; - match mailbox.enqueue_with_proactive_reference( - message, - &rbc_dag_committee_context, - proactive_reference, - ) { + match mailbox.enqueue(message, &event_inner.committee) { Ok(RbcDagOutboundEnqueueV1::Added) => shadow_metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["network", "sent"]) @@ -5580,31 +5503,21 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_coalesces_exact_duplicates_and_rejects_conflicts() { let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(committee).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); assert_eq!( mailbox - .enqueue( - rbc_dag_outbound_test_sync_response(7, 0xA1), - &committee_context, - ) + .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) .unwrap(), RbcDagOutboundEnqueueV1::Added ); assert_eq!( mailbox - .enqueue( - rbc_dag_outbound_test_sync_response(7, 0xA1), - &committee_context, - ) + .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) .unwrap(), RbcDagOutboundEnqueueV1::Coalesced ); assert!(matches!( - mailbox.enqueue( - rbc_dag_outbound_test_sync_response(7, 0xB1), - &committee_context, - ), + mailbox.enqueue(rbc_dag_outbound_test_sync_response(7, 0xB1), &committee), Err(RbcDagOutboundMailboxErrorV1::ConflictingDuplicate { class: RbcDagOutboundClassV1::Priority, key: RbcDagOutboundKeyV1::SyncResponse(1, 7), @@ -5618,12 +5531,11 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_drains_priority_before_proactive() { let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(committee.clone()).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); let (reference, proactive) = rbc_dag_outbound_test_carrier(&committee, 11); - mailbox.enqueue(proactive, &committee_context).unwrap(); + mailbox.enqueue(proactive, &committee).unwrap(); mailbox - .enqueue(rbc_dag_outbound_test_sync_request(9), &committee_context) + .enqueue(rbc_dag_outbound_test_sync_request(9), &committee) .unwrap(); let (class, first) = mailbox.try_pop().unwrap(); @@ -5638,9 +5550,7 @@ mod tests { let (class, second) = mailbox.try_pop().unwrap(); assert_eq!(class, RbcDagOutboundClassV1::Proactive); assert!(matches!( - rbc_dag_outbound_classification(&second, &committee_context, None) - .unwrap() - .1, + rbc_dag_outbound_classification(&second, &committee).unwrap().1, RbcDagOutboundKeyV1::Proactive(actual) if actual == reference )); } @@ -5648,7 +5558,6 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_never_evicts_a_unique_proactive_reference() { let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(committee.clone()).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); let mut first_reference = None; for marker in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY { @@ -5656,7 +5565,7 @@ mod tests { rbc_dag_outbound_test_carrier(&committee, marker as TimestampNs); first_reference.get_or_insert(reference); assert_eq!( - mailbox.enqueue(message, &committee_context).unwrap(), + mailbox.enqueue(message, &committee).unwrap(), RbcDagOutboundEnqueueV1::Added ); } @@ -5665,7 +5574,7 @@ mod tests { STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as TimestampNs + 1, ); assert!(matches!( - mailbox.enqueue(overflow, &committee_context), + mailbox.enqueue(overflow, &committee), Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { class: RbcDagOutboundClassV1::Proactive, capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, @@ -5687,14 +5596,10 @@ mod tests { #[test] fn rbc_dag_outbound_mailbox_bounds_distinct_priority_keys_and_bytes() { let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(committee).unwrap(); let mailbox = RbcDagOutboundMailboxV1::new(); for round in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber { mailbox - .enqueue( - rbc_dag_outbound_test_sync_request(round), - &committee_context, - ) + .enqueue(rbc_dag_outbound_test_sync_request(round), &committee) .unwrap(); } assert!(matches!( @@ -5702,7 +5607,7 @@ mod tests { rbc_dag_outbound_test_sync_request( STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber + 1, ), - &committee_context, + &committee, ), Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { class: RbcDagOutboundClassV1::Priority, @@ -5732,9 +5637,9 @@ mod tests { ), failure: None, }); - byte_bounded.enqueue(first, &committee_context).unwrap(); + byte_bounded.enqueue(first, &committee).unwrap(); assert!(matches!( - byte_bounded.enqueue(rbc_dag_outbound_test_sync_request(2), &committee_context,), + byte_bounded.enqueue(rbc_dag_outbound_test_sync_request(2), &committee), Err(RbcDagOutboundMailboxErrorV1::ByteCapacity { class: RbcDagOutboundClassV1::Priority, .. diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index ea8d2685..b89b0642 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -344,38 +344,10 @@ struct RbcCandidateState { votes: BTreeSet, acks: BTreeSet, readies: BTreeSet, - echo_stake: Stake, - vote_stake: Stake, - ack_stake: Stake, - ready_stake: Stake, requested_holders: BTreeSet, } impl RbcCandidateState { - fn insert_echo(&mut self, sender: AuthorityIndex, stake: Stake) { - if self.echoes.insert(sender) { - self.echo_stake = self.echo_stake.saturating_add(stake); - } - } - - fn insert_vote(&mut self, sender: AuthorityIndex, stake: Stake) { - if self.votes.insert(sender) { - self.vote_stake = self.vote_stake.saturating_add(stake); - } - } - - fn insert_ack(&mut self, sender: AuthorityIndex, stake: Stake) { - if self.acks.insert(sender) { - self.ack_stake = self.ack_stake.saturating_add(stake); - } - } - - fn insert_ready(&mut self, sender: AuthorityIndex, stake: Stake) { - if self.readies.insert(sender) { - self.ready_stake = self.ready_stake.saturating_add(stake); - } - } - fn holders(&self) -> BTreeSet { self.echoes .iter() @@ -1280,7 +1252,6 @@ impl RbcDagModel { fn authorize_local_echo(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; - let own_stake = self.authority_stake(own); if own == reference.authority { // The target author is excluded from ECHO/VOTE/ACK. A locally // fixed high-stake author instead seeds READY: its stake is at @@ -1299,7 +1270,8 @@ impl RbcDagModel { slot.candidates .entry(reference) .or_default() - .insert_echo(own, own_stake); + .echoes + .insert(own); let statement = RbcPhaseStatementV1::Echo { target: reference }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); @@ -1308,7 +1280,6 @@ impl RbcDagModel { fn authorize_local_ready(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; - let own_stake = self.authority_stake(own); let slot = self.rbc_slot_mut(reference); if slot.readied.is_some() { return; @@ -1318,7 +1289,8 @@ impl RbcDagModel { slot.candidates .entry(reference) .or_default() - .insert_ready(own, own_stake); + .readies + .insert(own); let statement = RbcPhaseStatementV1::Ready { target: reference }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); @@ -1434,7 +1406,6 @@ impl RbcDagModel { return; } } - let sender_stake = self.authority_stake(sender); let slot = self.rbc_slot_mut(target); let senders = match statement { RbcPhaseStatementV1::Echo { .. } => &mut slot.echo_by_sender, @@ -1452,16 +1423,16 @@ impl RbcDagModel { let candidate = slot.candidates.entry(target).or_default(); match statement { RbcPhaseStatementV1::Echo { .. } => { - candidate.insert_echo(sender, sender_stake); + candidate.echoes.insert(sender); } RbcPhaseStatementV1::Vote { .. } => { - candidate.insert_vote(sender, sender_stake); + candidate.votes.insert(sender); } RbcPhaseStatementV1::Ack { .. } => { - candidate.insert_ack(sender, sender_stake); + candidate.acks.insert(sender); } RbcPhaseStatementV1::Ready { .. } => { - candidate.insert_ready(sender, sender_stake); + candidate.readies.insert(sender); } } self.drive_rbc(target, log); @@ -1537,24 +1508,11 @@ impl RbcDagModel { .get(&slot_key) .and_then(|slot| slot.candidates.get(&target)) .expect("the candidate remains allocated"); - debug_assert_eq!( - candidate.echo_stake, - self.voters_stake_excluding(&candidate.echoes, target.authority) - ); - debug_assert_eq!( - candidate.vote_stake, - self.voters_stake_excluding(&candidate.votes, target.authority) - ); - debug_assert_eq!( - candidate.ack_stake, - self.voters_stake_excluding(&candidate.acks, target.authority) - ); - debug_assert_eq!(candidate.ready_stake, self.voters_stake(&candidate.readies)); ( - candidate.echo_stake, - candidate.vote_stake, - candidate.ack_stake, - candidate.ready_stake, + self.voters_stake_excluding(&candidate.echoes, target.authority), + self.voters_stake_excluding(&candidate.votes, target.authority), + self.voters_stake_excluding(&candidate.acks, target.authority), + self.voters_stake(&candidate.readies), ) }; let (vote_trigger, ack_trigger, optimistic_ready_trigger, promise_trigger) = thresholds @@ -1632,28 +1590,20 @@ impl RbcDagModel { } RbcAction::SendVote => { let own = self.own_authority; - let own_stake = self.authority_stake(own); let slot = self.rbc_slot_mut(target); slot.voted = Some(target); slot.vote_by_sender.insert(own, target); - slot.candidates - .entry(target) - .or_default() - .insert_vote(own, own_stake); + slot.candidates.entry(target).or_default().votes.insert(own); let statement = RbcPhaseStatementV1::Vote { target }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); } RbcAction::SendAck => { let own = self.own_authority; - let own_stake = self.authority_stake(own); let slot = self.rbc_slot_mut(target); slot.acked = Some(target); slot.ack_by_sender.insert(own, target); - slot.candidates - .entry(target) - .or_default() - .insert_ack(own, own_stake); + slot.candidates.entry(target).or_default().acks.insert(own); let statement = RbcPhaseStatementV1::Ack { target }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); @@ -3506,8 +3456,7 @@ mod tests { .carriers .insert(reference, CarrierRecord::new(carrier, false)); let mut candidate_state = RbcCandidateState::default(); - candidate_state.insert_ready(1, 1); - candidate_state.insert_ready(2, 1); + candidate_state.readies.extend([1, 2]); let mut slot = RbcSlotState::default(); slot.ready_by_sender .extend([(1, reference), (2, reference)]); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 24c80588..71950faa 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -1156,19 +1156,9 @@ impl CertifiedProjectionModel { let Some(projected) = self.vertices.get(&certifier) else { return false; }; - self.strong_parents_certify(projected.vertex.strong_parents(), leader) - } - - pub(crate) fn leader_values(&self, slot: LeaderSlotV1) -> Vec { - self.slot_values(slot.author, slot.round) - } - - fn strong_parents_certify( - &self, - strong_parents: &[ConsensusVertexReference], - leader: ConsensusVertexReference, - ) -> bool { - let voter_authors: BTreeSet<_> = strong_parents + let voter_authors: BTreeSet<_> = projected + .vertex + .strong_parents() .iter() .filter_map(|parent| { self.vertices.get(parent).and_then(|voter| { diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 321585a4..0e7e5737 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -26,9 +26,8 @@ use crate::{ RbcPhaseStatementV1, carrier_genesis_reference, journal::{IngressProvenanceV1, JournalErrorV1, JournalEventV1, WriteAheadJournalV1}, model::{ - DeliveryPromiseBasisV1, EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, - EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, ModelError, ModelInputRecord, - ModelTraceEvent, RbcDagModel, + DeliveryPromiseBasisV1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, ModelError, + ModelInputRecord, ModelTraceEvent, RbcDagModel, }, projection::{ C1StrongParentWitnessV1, CertifiedProjectionError, CertifiedProjectionModel, @@ -744,7 +743,6 @@ pub(crate) struct StarfishRbcDagShadowV1 { pending_projected_vertices: Vec, pending_projection_decisions: Vec, pending_committed_frontiers: Vec, - vote_qc_fast_path: bool, poisoned: bool, } @@ -792,7 +790,6 @@ impl StarfishRbcDagShadowV1 { authorizer, wal_sync_policy, ShadowFrontierRecoveryPolicyV1::Observational, - false, ) } @@ -808,7 +805,6 @@ impl StarfishRbcDagShadowV1 { authorizer: ShadowAuthorizerV1, wal_sync_policy: ShadowWalSyncPolicyV1, recovery_cursor: Option, - vote_qc_fast_path: bool, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { Self::open_with_frontier_recovery_policy( path, @@ -818,7 +814,6 @@ impl StarfishRbcDagShadowV1 { authorizer, wal_sync_policy, ShadowFrontierRecoveryPolicyV1::Authoritative(recovery_cursor), - vote_qc_fast_path, ) } @@ -831,7 +826,6 @@ impl StarfishRbcDagShadowV1 { authorizer: ShadowAuthorizerV1, wal_sync_policy: ShadowWalSyncPolicyV1, frontier_recovery_policy: ShadowFrontierRecoveryPolicyV1, - vote_qc_fast_path: bool, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { validate_configuration(&committee, own_authority, context, &authorizer)?; let committee_size = committee.committee().len(); @@ -880,7 +874,6 @@ impl StarfishRbcDagShadowV1 { pending_projected_vertices: Vec::new(), pending_projection_decisions: Vec::new(), pending_committed_frontiers: Vec::new(), - vote_qc_fast_path, poisoned: false, }; @@ -998,34 +991,12 @@ impl StarfishRbcDagShadowV1 { /// window. These are bounded by the reducer's future-carrier window and /// become admitted only through sequential clock advancement. pub(crate) fn buffered_authenticated_carrier_count(&self) -> usize { - let Some(first_buffered_round) = self - .local_carrier_round() - .checked_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) - .and_then(|round| round.checked_add(1)) - else { - return 0; - }; - let buffered = self - .committee - .committee() - .authorities() - .map(|authority| { - self.authenticated_slots - .range((authority, first_buffered_round)..=(authority, RoundNumber::MAX)) - .count() + self.authenticated_slots + .iter() + .filter(|((authority, round), reference)| { + self.model.admitted_reference(*authority, *round) != Some(**reference) }) - .sum(); - debug_assert_eq!( - buffered, - self.authenticated_slots - .iter() - .filter(|((authority, round), reference)| { - self.model.admitted_reference(*authority, *round) != Some(**reference) - }) - .count(), - "the reducer must admit every authenticated carrier inside its admission window" - ); - buffered + .count() } pub(crate) fn drain_projection_decisions(&mut self) -> Vec { @@ -2058,39 +2029,6 @@ impl StarfishRbcDagShadowV1 { Ok(()) } - /// Commit the longest consecutive prefix backed by an exact projected - /// vote quorum. Two conflicting vote quorums, or a vote quorum and a - /// negative-choice skip quorum, intersect in honest stake; an honest - /// logical author fixes exactly one choice. Reliable projected delivery - /// then makes the certificate eventually visible to every honest node. - /// This removes the redundant second certificate wave from the optimistic - /// direct-commit path while leaving skip and indirect recovery unchanged. - fn drive_vote_quorum_committer(&mut self) -> Result<(), ShadowErrorV1> { - loop { - let slot = self - .projection - .leader_slot(self.next_undecided_consensus_round); - let mut certified = Vec::new(); - for leader in self.projection.leader_values(slot) { - if self.projection.vote_stake(leader)? - >= self.committee.committee().quorum_threshold() - { - certified.push(leader); - } - } - if certified.len() > 1 { - return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues(slot).into()); - } - let Some(leader) = certified.pop() else { - return Ok(()); - }; - let decision = ProjectionDecisionV1::DirectCommit { leader }; - self.commit_projected_anchor(leader)?; - self.record_projection_decision(decision); - self.next_undecided_consensus_round = slot.round.saturating_add(1); - } - } - fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { if self.projected_decisions.insert(decision) { let slot = projection_decision_slot(decision); @@ -2422,11 +2360,7 @@ impl StarfishRbcDagShadowV1 { } self.activate_promised_references(); self.drive_promised_projection(); - self.drive_certified_projection()?; - if self.vote_qc_fast_path { - self.drive_vote_quorum_committer()?; - } - Ok(()) + self.drive_certified_projection() } fn decode_batch(&self, records: &[Vec]) -> Result { @@ -3863,45 +3797,6 @@ mod tests { [older, first_anchor, later_anchor] } - #[test] - fn vote_quorum_commits_before_the_certifier_round_projects() { - let mut network = TestNetwork::new(); - let node = &mut network.nodes[0]; - assert!(!node.vote_qc_fast_path, "strict finality is the default"); - node.vote_qc_fast_path = true; - let leader_author = node.committee.committee().elect_leader(1); - let leader = ordered_committer_vertex(leader_author, 1, 0); - node.projection.inject_projected_for_test( - leader, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author, - leader_round: 0, - }, - ); - for author in 0..3 as AuthorityIndex { - node.projection.inject_projected_for_test( - ordered_committer_vertex(author, 2, 0), - vec![leader], - LeaderChoiceV1::Vote { leader }, - ); - } - - let slot = node.projection.leader_slot(1); - assert_eq!( - node.projection.direct_decision(slot).unwrap(), - ProjectionDecisionV1::Undecided { slot }, - "the legacy two-level rule still waits for round-three certifiers" - ); - node.drive_vote_quorum_committer().unwrap(); - assert_eq!( - node.drain_projection_decisions(), - vec![ProjectionDecisionV1::DirectCommit { leader }] - ); - assert_eq!(node.drain_committed_frontiers().len(), 1); - assert_eq!(node.next_undecided_consensus_round, 2); - } - #[test] fn ordered_committer_is_deterministic_across_direct_anchor_arrival_orders() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index d9ad8613..389ee9a4 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -64,23 +64,20 @@ use crate::{ // A mirror run must absorb one complete committee fan-in plus a small reserve; // autonomous repair additionally budgets a simultaneous request and response -// per peer. At the four-MiB carrier cap, allowing at most 128 queued inputs -// caps carrier payload retention at 512 MiB (plus bounded sidecars and -// allocator overhead). This permits 124 mirror validators or 42 autonomous -// validators, including the 40-validator comparison profile. Larger committees -// are rejected for this benchmark prototype +// per peer. At the four-MiB carrier cap, allowing at most 64 queued inputs also +// caps carrier payload retention at 256 MiB (plus bounded sidecars and +// allocator overhead). This permits 60 mirror validators or 20 autonomous +// validators. Larger committees are rejected for this benchmark prototype // instead of silently under-sizing the queue and reporting incomparable // results. // Use the full bounded allowance even for a small committee. A single fan-in // reserve is insufficient when several round bursts arrive while the actor is // synchronously making the previous transition durable. const SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1: usize = 64; -const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 128; +const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 64; const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); -const SHADOW_APPLICATION_SUBMISSION_GRACE_V1: Duration = Duration::from_millis(100); -const SHADOW_APPLICATION_SUBMISSION_GRACE_COMMITTEE_STEP_V1: usize = 10; const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); const SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1: Duration = Duration::from_millis(100); // At most sixteen distinct exact slots may bypass the duplicate retry interval @@ -99,12 +96,7 @@ const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_milli /// frame ceiling is larger; a dedicated configurable payload limit remains a /// deployment-hardening boundary. Keeping only a bounded recent window stops /// unsolicited sidecars from turning the actor into an unbounded cache. -// Bound the materialized-payload/quarantine cache independently from history, -// but size it for the same three-message-per-peer fan-in that the autonomous -// actor accepts. A fixed 64-entry callback map is insufficient at n=40: two -// adjacent carrier waves can complete verification while the actor applies -// the previous wave. The global 128 ceiling keeps the testbed bound explicit. -const SHADOW_APPLICATION_PAYLOAD_MIN_CAPACITY_V1: usize = 64; +const SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1: usize = SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1; const SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1: usize = MAX_CARRIER_CONTENT_SIZE_V1; const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); /// Bound healthy physical-carrier production independently of actor/network @@ -114,22 +106,6 @@ const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_mi const SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1: u32 = 20; const SHADOW_NORMAL_CARRIER_MIN_SPACING_V1: Duration = Duration::from_millis(1); -fn shadow_application_payload_capacity(committee_size: usize) -> usize { - committee_size.saturating_mul(3).clamp( - SHADOW_APPLICATION_PAYLOAD_MIN_CAPACITY_V1, - SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1, - ) -} - -fn shadow_application_submission_grace(committee_size: usize) -> Duration { - let fan_in_groups = committee_size - .max(1) - .div_ceil(SHADOW_APPLICATION_SUBMISSION_GRACE_COMMITTEE_STEP_V1); - SHADOW_APPLICATION_SUBMISSION_GRACE_V1 - .checked_div(u32::try_from(fan_in_groups).unwrap_or(u32::MAX)) - .unwrap_or_default() -} - type CarrierSyncSlotV1 = (RoundNumber, AuthorityIndex); type DesiredCarrierSyncResponseV1 = (AuthorityIndex, RbcDagShadowCarrierSyncResponse); @@ -462,9 +438,10 @@ impl StarfishRbcDagShadowServiceHandleV1 { existing.acknowledge_assignment |= local.acknowledge_assignment; return Ok(()); } - let capacity = shadow_application_payload_capacity(self.committee_size); - if desired.len() >= capacity { - return Err(ShadowServiceErrorV1::ApplicationStateCapacity { capacity }); + if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + return Err(ShadowServiceErrorV1::ApplicationStateCapacity { + capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, + }); } desired.insert(local.round, local); drop(desired); @@ -761,9 +738,10 @@ impl StarfishRbcDagShadowServiceHandleV1 { } return Ok(()); } - let capacity = shadow_application_payload_capacity(self.committee_size); - if desired.len() >= capacity { - return Err(ShadowServiceErrorV1::ApplicationStateCapacity { capacity }); + if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { + return Err(ShadowServiceErrorV1::ApplicationStateCapacity { + capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, + }); } desired.insert(application, payload); drop(desired); @@ -1249,7 +1227,6 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( None, None, true, - false, ) } @@ -1284,7 +1261,6 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( Some(metrics), None, true, - false, ) } @@ -1322,7 +1298,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( None, None, true, - false, ) } @@ -1361,7 +1336,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( Some(metrics), None, true, - false, ) } @@ -1404,7 +1378,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_paused_with_metric Some(metrics), None, false, - false, ) } @@ -1426,7 +1399,6 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 metrics: Arc, recovery_cursor: Option, clock_starts_active: bool, - vote_qc_fast_path: bool, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -1454,7 +1426,6 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 Some(metrics), recovery_cursor, clock_starts_active, - vote_qc_fast_path, ) } @@ -1524,7 +1495,6 @@ fn spawn_consensus_timeout_deadline_task( }); } -#[allow(clippy::too_many_arguments)] fn start_starfish_rbc_dag_shadow_service_with_mode_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, @@ -1538,7 +1508,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( metrics: Option>, recovery_cursor: Option, clock_starts_active: bool, - vote_qc_fast_path: bool, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -1720,7 +1689,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( authorizer, wal_sync_policy, recovery_cursor, - vote_qc_fast_path, ) } else { StarfishRbcDagShadowV1::open_with_wal_sync_policy( @@ -1926,7 +1894,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( for (carrier, header) in recovered_application_headers .into_iter() .rev() - .take(shadow_application_payload_capacity(committee_size)) + .take(SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1) { authorized_applications.insert( header.reference(), @@ -1993,7 +1961,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( #[cfg(test)] sync_max_desired_responses: 0, awaiting_application_submission: false, - application_submission_deadline: None, consensus_pacemaker, consensus_timeout, normal_carrier_min_spacing: mode @@ -2264,13 +2231,10 @@ struct ShadowServiceStateV1 { #[cfg(test)] sync_max_desired_responses: usize, /// A successful application-carrier assignment just released the core's - /// one-outstanding producer gate. Give that exact producer a bounded - /// maintenance epoch to submit its successor before any normal phase or - /// fallback work spends the next physical slot on a control carrier. - /// Repair traffic remains independently bounded and may bypass this - /// scheduling preference. + /// one-outstanding producer gate. Give that exact producer one actor turn + /// to submit its successor before C3 spends the next physical slot on a + /// control heartbeat. Maintenance/heartbeat messages bound the wait. awaiting_application_submission: bool, - application_submission_deadline: Option, consensus_pacemaker: ConsensusPacemakerV1, /// Logical C2 fallback deadline. This is independent of the physical /// heartbeat so experiments can vary consensus permission without @@ -2315,7 +2279,6 @@ impl ShadowServiceStateV1 { self.consensus_pacemaker = ConsensusPacemakerV1::new(self.core.next_local_consensus_round()); self.awaiting_application_submission = false; - self.application_submission_deadline = None; self.heartbeat_notification_pending .store(false, Ordering::Release); @@ -2426,9 +2389,8 @@ impl ShadowServiceStateV1 { } fn make_application_state_room(&mut self, application: BlockReference) { - let capacity = shadow_application_payload_capacity(self.committee_size); if self.authorized_applications.contains_key(&application) - || self.authorized_applications.len() < capacity + || self.authorized_applications.len() < SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { return; } @@ -2520,8 +2482,7 @@ impl ShadowServiceStateV1 { payload: Option>, ) -> Result<(), ShadowServiceErrorV1> { if !self.quarantined_application_payloads.contains_key(&carrier) - && self.quarantined_application_payloads.len() - >= shadow_application_payload_capacity(self.committee_size) + && self.quarantined_application_payloads.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { self.quarantined_application_payloads.pop_first(); } @@ -2797,7 +2758,6 @@ impl ShadowServiceStateV1 { let desired = std::mem::take(&mut *self.desired_local_applications.lock()); if !desired.is_empty() { self.awaiting_application_submission = false; - self.application_submission_deadline = None; } for local in desired.into_values() { self.enqueue_local(local); @@ -3267,7 +3227,6 @@ impl ShadowServiceStateV1 { return; } self.awaiting_application_submission = false; - self.application_submission_deadline = None; } fn cancel_normal_carrier_deadline(&mut self) { @@ -3461,8 +3420,6 @@ impl ShadowServiceStateV1 { self.report_wal_delta(before); if let Some(reference) = assigned_application { self.awaiting_application_submission = true; - self.application_submission_deadline = Instant::now() - .checked_add(shadow_application_submission_grace(self.committee_size)); self.emit(ShadowServiceEventV1::ApplicationAssigned(reference)); } if let Some(application) = application { @@ -3671,25 +3628,6 @@ impl ShadowServiceStateV1 { /// harmless idempotent replays. fn retry_pending_local(&mut self) { if self.mode.is_autonomous() { - // The producer is released only after its previous application - // carrier is durably fixed. Phase effects from that transition - // are processed synchronously and would otherwise consume the - // newly opened physical slot before the producer's successor can - // cross the ordered Core bridge. Piggyback those phases on the - // successor when it arrives. A deadline anchored at assignment - // retries the phase carrier after 100 ms, so a stalled producer - // cannot stop physical RBC progress. - if self.awaiting_application_submission && self.pending_local.is_empty() { - let now = Instant::now(); - if let Some(deadline) = self.application_submission_deadline { - if now < deadline { - self.schedule_normal_carrier_deadline(deadline); - return; - } - } - self.awaiting_application_submission = false; - self.application_submission_deadline = None; - } while (!self.pending_local.is_empty() || self.core.has_pending_application_phase_work()) && self.core.can_create_carrier() && !self.fatal @@ -4638,8 +4576,8 @@ fn run_shadow_service( } ShadowServiceMessageV1::TopologyChanged => {} ShadowServiceMessageV1::RetryRecovery => { + state.awaiting_application_submission = false; state.reconcile_local_applications(); - state.retry_pending_local(); state.reconcile_pending_recovery(); state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); @@ -4647,7 +4585,6 @@ fn run_shadow_service( } ShadowServiceMessageV1::HeartbeatTick => { state.awaiting_application_submission = false; - state.application_submission_deadline = None; state.try_create_autonomous_carrier(); } ShadowServiceMessageV1::NormalCarrierDeadline { generation } => { @@ -5827,7 +5764,6 @@ mod tests { None, None, false, - false, ) .unwrap() } @@ -5919,7 +5855,6 @@ mod tests { sync_max_outstanding: 0, sync_max_desired_responses: 0, awaiting_application_submission: false, - application_submission_deadline: None, consensus_pacemaker: ConsensusPacemakerV1::new(slot), consensus_timeout: leader_timeout, // Existing state-level tests invoke creation synchronously and do @@ -6846,8 +6781,7 @@ mod tests { ); tokio::task::spawn_blocking(move || { let mut state = state; - let capacity = shadow_application_payload_capacity(N); - for offset in 0..=capacity { + for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { let round = offset as RoundNumber + 1; let header = direct_header(0, round, offset as u8); state @@ -6861,9 +6795,12 @@ mod tests { ) .unwrap(); } - assert_eq!(state.authorized_applications.len(), capacity); + assert_eq!( + state.authorized_applications.len(), + SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + ); - for offset in 0..=capacity { + for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { let round = offset as RoundNumber + 1; state .quarantine_application( @@ -6874,7 +6811,10 @@ mod tests { ) .unwrap(); } - assert_eq!(state.quarantined_application_payloads.len(), capacity); + assert_eq!( + state.quarantined_application_payloads.len(), + SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 + ); state.core.shutdown().unwrap(); }) .await @@ -6900,7 +6840,7 @@ mod tests { tokio::task::spawn_blocking(move || { let mut state = state; let mut applications = Vec::new(); - for offset in 0..=shadow_application_payload_capacity(N) { + for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { let round = offset as RoundNumber + 1; let header = direct_header(0, round, offset as u8); applications.push(header.reference()); @@ -7258,47 +7198,6 @@ mod tests { state.core.shutdown().unwrap(); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn application_submission_grace_is_anchored_bounded_and_coalesced() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (mut state, _events, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - - let round = state.core.local_carrier_round(); - let deadline = Instant::now() + Duration::from_secs(60); - state.awaiting_application_submission = true; - state.application_submission_deadline = Some(deadline); - state.retry_pending_local(); - assert_eq!(state.core.local_carrier_round(), round); - assert_eq!( - state - .normal_carrier_deadline - .expect("the grace must reuse the coalesced carrier wake") - .deadline, - deadline - ); - - let generation = state.normal_carrier_generation; - state.retry_pending_local(); - assert_eq!(state.normal_carrier_generation, generation); - state.application_submission_deadline = Some(Instant::now()); - state.retry_pending_local(); - assert!(!state.awaiting_application_submission); - assert_eq!(state.application_submission_deadline, None); - state.core.shutdown().unwrap(); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn paused_autonomous_clock_uses_one_idempotent_fresh_activation_epoch() { let harness = Harness::new(); @@ -9089,8 +8988,8 @@ mod tests { } #[tokio::test] - async fn maximum_mirror_burst_fits_before_the_actor_drains() { - const LARGE_N: usize = 124; + async fn sixty_validator_burst_fits_before_the_actor_drains() { + const LARGE_N: usize = 60; let input_capacity = shadow_input_capacity(LARGE_N, ShadowServiceModeV1::DirectMirror).unwrap(); assert_eq!(input_capacity, SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1); @@ -9134,33 +9033,17 @@ mod tests { } #[test] - fn autonomous_burst_budget_accepts_forty_two_and_rejects_forty_three() { + fn autonomous_burst_budget_accepts_twenty_and_rejects_twenty_one() { let mode = ShadowServiceModeV1::AutonomousClock { heartbeat_interval: Duration::from_millis(250), }; - assert_eq!(shadow_application_payload_capacity(10), 64); - assert_eq!(shadow_application_payload_capacity(40), 120); - assert_eq!(shadow_application_payload_capacity(42), 126); - assert_eq!( - shadow_application_submission_grace(10), - Duration::from_millis(100) - ); - assert_eq!( - shadow_application_submission_grace(40), - Duration::from_millis(25) - ); - assert_eq!( - shadow_application_submission_grace(42), - Duration::from_millis(20) - ); - assert_eq!(shadow_input_capacity(40, mode).unwrap(), 122); - assert_eq!(shadow_input_capacity(42, mode).unwrap(), 128); + assert_eq!(shadow_input_capacity(20, mode).unwrap(), 64); assert!(matches!( - shadow_input_capacity(43, mode), + shadow_input_capacity(21, mode), Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { - committee_size: 43, - required_capacity: 131, - maximum_capacity: 128, + committee_size: 21, + required_capacity: 65, + maximum_capacity: 64, }) )); } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 57873315..49eebc22 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -134,22 +134,6 @@ impl Validator { "Starfish-RBC-DAG embedded RBC authority requires the autonomous RBC-DAG shadow" )); } - if public_config.parameters.starfish_rbc_dag_vote_qc_fast_path - && !public_config - .parameters - .starfish_rbc_dag_embedded_rbc_authority - { - return Err(eyre!( - "Starfish-RBC-DAG vote-QC fast path requires embedded RBC-DAG authority" - )); - } - if public_config.parameters.starfish_rbc_dag_vote_qc_fast_path - && parameters.benchmark_duration.is_none() - { - return Err(eyre!( - "Starfish-RBC-DAG vote-QC fast path is restricted to finite testbed benchmarks" - )); - } if public_config .parameters .starfish_rbc_single_dag_echo_qc_fast_path @@ -169,7 +153,7 @@ impl Validator { { return Err(eyre!( "Starfish-RBC single-DAG ECHO-QC fast path is restricted to finite testbed \ - benchmarks" + benchmarks" )); } if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { @@ -593,67 +577,6 @@ mod smoke_tests { })); } - #[tokio::test] - async fn vote_qc_fast_path_requires_embedded_authority() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_vote_qc_fast_path = true; - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish-rbc".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("vote-QC fast path requires embedded RBC-DAG authority") - })); - } - - #[tokio::test] - async fn vote_qc_fast_path_requires_a_finite_benchmark() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_shadow = true; - public_config.parameters.starfish_rbc_dag_autonomous_clock = true; - public_config - .parameters - .starfish_rbc_dag_embedded_rbc_authority = true; - public_config.parameters.starfish_rbc_dag_vote_qc_fast_path = true; - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish-rbc".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("vote-QC fast path is restricted to finite testbed benchmarks") - })); - } - #[tokio::test] async fn buffered_shadow_wal_requires_shadow_mode() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 0b3f1481..96c9c0f2 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -205,11 +205,6 @@ enum Operation { /// headers. Requires the autonomous RBC-DAG mode. #[clap(long, default_value_t = false)] starfish_rbc_dag_embedded_rbc_authority: bool, - /// Testbed-only: directly commit a projected leader after its exact - /// projected vote quorum, skipping the strict second certifier wave. - /// Requires embedded RBC-DAG authority and changes the finality proof. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_vote_qc_fast_path: bool, /// Testbed-only: deliver a single-DAG RBC header after a receiver-local /// quorum ECHO. This preserves uniqueness but not Byzantine /// selective-withholding totality, so it is restricted to finite @@ -342,7 +337,6 @@ async fn main() -> Result<()> { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, - starfish_rbc_dag_vote_qc_fast_path, starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, @@ -361,7 +355,6 @@ async fn main() -> Result<()> { node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; node_parameters.starfish_rbc_dag_embedded_rbc_authority = starfish_rbc_dag_embedded_rbc_authority; - node_parameters.starfish_rbc_dag_vote_qc_fast_path = starfish_rbc_dag_vote_qc_fast_path; node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = starfish_rbc_single_dag_echo_qc_fast_path; node_parameters.starfish_rbc_dag_consensus_timeout = @@ -606,14 +599,6 @@ async fn local_benchmark( .unwrap_or(node_parameters.leader_timeout) .as_millis() ); - println!( - "Vote-QC direct commit: {}", - if node_parameters.starfish_rbc_dag_vote_qc_fast_path { - "ENABLED (testbed-only; strict certifier wave skipped)" - } else { - "disabled (strict two-level Starfish finality)" - } - ); } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( @@ -1558,7 +1543,6 @@ mod tests { "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-embedded-rbc-authority", - "--starfish-rbc-dag-vote-qc-fast-path", "--starfish-rbc-single-dag-echo-qc-fast-path", "--starfish-rbc-dag-consensus-timeout-ms", "250", @@ -1574,7 +1558,6 @@ mod tests { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, - starfish_rbc_dag_vote_qc_fast_path, starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, @@ -1589,7 +1572,6 @@ mod tests { assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_dag_embedded_rbc_authority); - assert!(starfish_rbc_dag_vote_qc_fast_path); assert!(starfish_rbc_single_dag_echo_qc_fast_path); assert_eq!(starfish_rbc_dag_consensus_timeout_ms, Some(250)); assert!(starfish_rbc_dag_shadow_buffered_wal); diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 1fbe5b3c..63fefb13 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -720,13 +720,6 @@ The existing Starfish patterns are then evaluated from these explicit choices: - `Q` distinct certifier authors provide the direct-commit condition; and - a per-candidate quorum of explicit negative choices provides the direct-skip pattern. -This two-level rule is the strict default. The local testbed can explicitly enable -`--starfish-rbc-dag-vote-qc-fast-path`, which directly commits an exact projected leader after the -preceding projected vote quorum and therefore skips the second certifier wave. The experiment adds -no wire messages, retains the ordinary skip and indirect-recovery paths, and is intentionally -labelled separately because it changes the proof shape described by this section; it is not a -production-finality claim. - If the leader produces no value, `Q` immutable `NoVote(slot)` choices are a self-contained direct skip witness. If a Byzantine leader equivocates, `Vote(L)` is negative evidence for every other candidate, and the current Starfish per-candidate evaluator decides whether the collected explicit @@ -884,12 +877,12 @@ newest current-process observation (`<= 4`). These are empirical benchmark cover asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as proof of a protocol failure. -The actor reserves a bounded queue of up to 128 entries so several fan-in bursts can wait behind a slow +The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a slow reference transition (including synchronous fsync in the crash-safe profile), capping queued -maximum-sized carrier bodies at 512 MiB (plus sidecars and allocator overhead). Mirror mode budgets +maximum-sized carrier bodies at 256 MiB (plus sidecars and allocator overhead). Mirror mode budgets one peer fan-in plus five local/control inputs and accepts -at most 124 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and -exact-slot response per peer plus five control inputs and accepts at most 42 validators. Larger runs +at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and +exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, healthy proactive rounds receive a repair grace period, and exact synchronization is rate-limited per peer. Exact synchronization transfers only one requested `(author, round)` and only from the From 29d020bd36f131e855298dc96c2dc6eb970dd450 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:37:40 +0200 Subject: [PATCH 52/62] Revert "Instrument RBC-DAG consensus cadence" This reverts commit 9cdbc16776a90199ed8f6a41a3f169a24c9f4cbb. --- crates/starfish-core/src/config.rs | 15 - crates/starfish-core/src/metrics.rs | 29 -- crates/starfish-core/src/net_sync.rs | 27 -- crates/starfish-core/src/network.rs | 31 -- .../src/starfish_rbc_dag_shadow.rs | 154 ------ .../src/starfish_rbc_dag_shadow_service.rs | 440 +----------------- crates/starfish-core/src/validator.rs | 51 -- crates/starfish/src/main.rs | 18 - docs/starfish-rbc-dag-protocol.md | 56 +-- 9 files changed, 24 insertions(+), 797 deletions(-) diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 9e9c3749..0c10344d 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -76,13 +76,6 @@ pub struct NodeParameters { /// This remains experimental and requires `starfish_rbc_dag_shadow`. #[serde(default)] pub starfish_rbc_dag_autonomous_clock: bool, - /// Optional logical C2 fallback timeout for the autonomous RBC-DAG. - /// `None` preserves the historical behavior and reuses `leader_timeout`. - /// This is deliberately independent of the physical carrier heartbeat so - /// latency experiments can vary consensus fallback without changing the - /// proactive push cadence. - #[serde(default)] - pub starfish_rbc_dag_consensus_timeout: Option, /// Use embedded carrier ECHO/READY delivery as the certification authority /// for Starfish-RBC application headers. Direct RBC retains INIT/payload /// transport but its phase messages cannot mark a block clean. @@ -178,7 +171,6 @@ impl Default for NodeParameters { starfish_rbc_protocol_instance: None, starfish_rbc_dag_shadow: false, starfish_rbc_dag_autonomous_clock: false, - starfish_rbc_dag_consensus_timeout: None, starfish_rbc_dag_embedded_rbc_authority: false, starfish_rbc_single_dag_echo_qc_fast_path: false, starfish_rbc_dag_shadow_buffered_wal: false, @@ -454,7 +446,6 @@ mod tests { assert_eq!(parameters.starfish_rbc_protocol_instance, None); assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); - assert_eq!(parameters.starfish_rbc_dag_consensus_timeout, None); assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); @@ -469,7 +460,6 @@ mod tests { ); assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); - assert_eq!(decoded.starfish_rbc_dag_consensus_timeout, None); assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); } @@ -480,7 +470,6 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, leader_timeout: Duration::from_millis(125), - starfish_rbc_dag_consensus_timeout: Some(Duration::from_millis(75)), starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -491,10 +480,6 @@ mod tests { assert!(decoded.starfish_rbc_dag_autonomous_clock); assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(decoded.leader_timeout, Duration::from_millis(125)); - assert_eq!( - decoded.starfish_rbc_dag_consensus_timeout, - Some(Duration::from_millis(75)) - ); } #[test] diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 564fdfd9..f118bc4a 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -2431,35 +2431,6 @@ impl Metrics { .unwrap_or_default(), ) ]); - table.add_row(row![ - b->"Logical vertex creation:", - format!( - "bootstrap/C1/C2/C3/omitted={}/{}/{}/{}/{}", - shadow_input_count("consensus_vertex", "bootstrap"), - shadow_input_count("consensus_vertex", "c1"), - shadow_input_count("consensus_vertex", "c2"), - shadow_input_count("consensus_vertex", "c3"), - shadow_input_count("consensus_vertex", "omitted"), - ) - ]); - let consensus_carrier_breakdown = |kind: &str| { - format!( - "bootstrap/C1/C2/C3/omitted={}/{}/{}/{}/{}", - shadow_input_count(kind, "bootstrap"), - shadow_input_count(kind, "c1"), - shadow_input_count(kind, "c2"), - shadow_input_count(kind, "c3"), - shadow_input_count(kind, "omitted"), - ) - }; - table.add_row(row![ - b->"Logical vertices by carrier kind:", - format!( - "application [{}], control/phase [{}]", - consensus_carrier_breakdown("application_consensus_vertex"), - consensus_carrier_breakdown("control_consensus_vertex"), - ) - ]); let stage_latency = RBC_DAG_PIPELINE_LATENCY_STAGES .iter() .map(|stage| { diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index f7765ad0..e170a779 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -109,7 +109,6 @@ enum RbcDagOutboundKeyV1 { Proactive(BlockReference), CarrierRequest(BlockReference), CarrierResponse(BlockReference), - CarrierEnvelopeResponse(BlockReference), SyncRequest(AuthorityIndex, RoundNumber), SyncResponse(AuthorityIndex, RoundNumber), ApplicationPayloadRequest(BlockReference), @@ -482,10 +481,6 @@ fn rbc_dag_outbound_classification( priority, RbcDagOutboundKeyV1::CarrierResponse(response.reference), )), - NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(response) => Ok(( - priority, - RbcDagOutboundKeyV1::CarrierEnvelopeResponse(response.reference), - )), NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => Ok(( priority, RbcDagOutboundKeyV1::SyncRequest(request.author, request.round), @@ -519,10 +514,6 @@ fn rbc_dag_outbound_messages_equal(left: &NetworkMessage, right: &NetworkMessage NetworkMessage::RbcDagShadowCarrierResponse(left), NetworkMessage::RbcDagShadowCarrierResponse(right), ) => left == right, - ( - NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(left), - NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(right), - ) => left == right, ( NetworkMessage::RbcDagShadowCarrierSyncRequest(left), NetworkMessage::RbcDagShadowCarrierSyncRequest(right), @@ -1747,21 +1738,6 @@ impl ConnectionHandler { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .carrier_envelope_response_reliably(self.peer_id, response) - .await - { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); - } - tracing::warn!( - "Failed to forward RBC-DAG shadow envelope response: {error}" - ); - } - } - } NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow @@ -3254,9 +3230,6 @@ impl NetworkSyncer // resolved Starfish leader timeout. Application and // embedded RBC phase carriers remain event-driven. node_parameters.leader_timeout, - node_parameters - .starfish_rbc_dag_consensus_timeout - .unwrap_or(node_parameters.leader_timeout), wal_sync_policy, Arc::clone(&metrics), rbc_dag_frontier_recovery_cursor, diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 63799242..86856ae5 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -123,18 +123,6 @@ pub struct RbcDagShadowCarrierResponse { pub canonical_carrier: Vec, } -/// Canonical carrier content plus one exact authentication-sidecar variant -/// retained by a phase-evidence holder. The requester recomputes `reference` -/// and verifies only its receiver-specific entry. A valid entry grants the -/// same authority as ordinary relayed ingress; an invalid entry falls back to -/// content-only recovery without blaming the author or holder. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct RbcDagShadowCarrierEnvelopeResponse { - pub reference: BlockReference, - pub canonical_carrier: Vec, - pub authentication_sidecar: Vec, -} - /// Request one exact carrier-clock slot from a peer. Keeping synchronization /// slot-addressed prevents an untrusted peer from choosing an unbounded range /// of history to return. @@ -291,10 +279,6 @@ pub enum NetworkMessage { /// Return commitment-checked transaction data for an authorized embedded /// application header. The response is not an author proof. RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse), - /// Return canonical carrier content with one exact retained - /// authentication-sidecar variant. Appended after the frozen V1 message - /// family so every preceding bincode enum discriminant remains stable. - RbcDagShadowCarrierEnvelopeResponse(RbcDagShadowCarrierEnvelopeResponse), } impl NetworkMessage { @@ -329,9 +313,6 @@ impl NetworkMessage { Self::RbcDagShadowCarrierSyncResponse(_) => "rbc_dag_shadow_carrier_sync_response", Self::RbcDagApplicationPayloadRequest(_) => "rbc_dag_application_payload_request", Self::RbcDagApplicationPayloadResponse(_) => "rbc_dag_application_payload_response", - Self::RbcDagShadowCarrierEnvelopeResponse(_) => { - "rbc_dag_shadow_carrier_envelope_response" - } } } } @@ -1963,13 +1944,6 @@ mod tests { reference: block_ref, canonical_carrier: vec![0xA6, 0xA7], }); - let shadow_envelope_response = NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( - RbcDagShadowCarrierEnvelopeResponse { - reference: block_ref, - canonical_carrier: vec![0xB6, 0xB7], - authentication_sidecar: vec![0xB8, 0xB9], - }, - ); let sync_request = NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { author: 2, @@ -2001,11 +1975,6 @@ mod tests { (sync_response, 19, "rbc_dag_shadow_carrier_sync_response"), (payload_request, 20, "rbc_dag_application_payload_request"), (payload_response, 21, "rbc_dag_application_payload_response"), - ( - shadow_envelope_response, - 22, - "rbc_dag_shadow_carrier_envelope_response", - ), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 0e7e5737..8cb806d7 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -1340,20 +1340,6 @@ impl StarfishRbcDagShadowV1 { self.next_local_consensus_round } - /// Whether the currently open logical slot can be fixed through the - /// optimistic C1 path using only already-promised projection state. This - /// is a pure preview of the same builder used for durable carrier - /// creation; callers must recompute when they actually create. - pub(crate) fn local_consensus_vertex_c1_ready(&self) -> bool { - if self.next_local_consensus_round <= 1 { - return false; - } - let Ok((own_prev, _)) = self.model.local_parent_set() else { - return false; - }; - self.build_local_consensus_vertex(own_prev, false).is_some() - } - pub(crate) fn projected_consensus_stake(&self, round: RoundNumber) -> Stake { self.promised_projection.projected_stake_at_round(round) } @@ -1618,55 +1604,6 @@ impl StarfishRbcDagShadowV1 { self.apply_requested_recovery(candidate) } - /// Recover an exact phase-evidenced carrier through the ordinary - /// receiver-bound authentication predicate. A valid sidecar variant is - /// durably recorded as relayed/direct authenticated ingress; an invalid - /// variant retains the exact requested content through the existing - /// content-only recovery path. MAC failures assign no blame. - pub(crate) fn recover_or_admit_from_peer( - &mut self, - expected_reference: BlockReference, - canonical_carrier_wire: &[u8], - authentication_sidecar: &[u8], - trusted_peer: AuthorityIndex, - ) -> Result { - self.ensure_live()?; - if !self.committee.committee().known_authority(trusted_peer) { - return Err(ShadowErrorV1::UnknownAuthority(trusted_peer)); - } - if !self.requested_recoveries.contains_key(&expected_reference) { - return Err(ShadowErrorV1::UnrequestedRecovery(expected_reference)); - } - let candidate = decode_candidate( - canonical_carrier_wire, - &self.committee, - Some(expected_reference), - )?; - let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); - validate_provenance(provenance, candidate.header().author(), &self.committee)?; - - match self.authenticate_decoded(candidate.clone(), authentication_sidecar) { - Ok(authenticated) => { - let (effects, applied) = - self.apply_authenticated_capability(authenticated, provenance)?; - if applied { - return Ok(ShadowIngressOutcomeV1::new( - ShadowIngressDispositionV1::Authenticated, - effects, - )); - } - } - Err(ShadowErrorV1::Carrier(_)) | Err(ShadowErrorV1::NonCanonicalAuthentication) => {} - Err(error) => return Err(error), - } - - let effects = self.apply_requested_recovery(candidate)?; - Ok(ShadowIngressOutcomeV1::new( - ShadowIngressDispositionV1::CandidateRetained, - effects, - )) - } - pub(crate) fn retained_candidate_wire(&self, reference: BlockReference) -> Option> { self.journal .snapshot() @@ -1674,31 +1611,6 @@ impl StarfishRbcDagShadowV1 { .map(<[u8]>::to_vec) } - /// Return one exact, durably retained authentication variant for a - /// carrier. Locally authored envelopes use the outbound WAL record; - /// received envelopes use the first authenticated ingress record, whose - /// provenance and bytes replay identically after restart. - pub(crate) fn retained_authenticated_envelope( - &self, - reference: BlockReference, - ) -> Option { - if reference.authority == self.own_authority { - return self - .local_outbound_envelope(reference.round) - .filter(|envelope| envelope.reference() == reference); - } - self.journal - .snapshot() - .authenticated_ingress() - .iter() - .find(|ingress| ingress.reference() == reference) - .map(|ingress| ShadowOutboundEnvelopeV1 { - reference, - canonical_carrier_wire: ingress.canonical_carrier_wire().to_vec(), - authentication_sidecar: ingress.authentication_sidecar().to_vec(), - }) - } - /// Decode canonical carrier bytes without mutating the reducer. Sync /// clients use this to bind a response to the requested author and round /// before passing it through normal authenticated ingress. @@ -5012,72 +4924,6 @@ mod tests { assert_eq!(network.nodes[0].wal_counts(), (0, 0)); } - #[test] - fn relayed_authenticated_envelope_reopens_with_exact_provenance_and_bytes() { - let mut network = TestNetwork::new(); - let candidate = round_one_candidate(1, &network.committee, 0x80); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - let wire = candidate.canonical_wire_bytes().unwrap(); - let sidecar = authentication.canonical_wire_bytes(); - network.nodes[0] - .receive_authenticated_from_peer(&wire, &sidecar, 2) - .unwrap(); - assert_eq!( - network.nodes[0].admitted_reference(1, 1), - Some(candidate.reference()) - ); - assert_eq!( - network.nodes[0].journal.snapshot().authenticated_ingress()[0].provenance(), - IngressProvenanceV1::Relayed { peer: 2 } - ); - let retained = network.nodes[0] - .retained_authenticated_envelope(candidate.reference()) - .unwrap(); - assert_eq!(retained.canonical_carrier_wire(), wire); - assert_eq!(retained.authentication_sidecar(), sidecar); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let (reopened, report) = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap(); - assert_eq!(report.replayed_batches(), 1); - assert_eq!( - reopened.authenticated_reference(1, 1), - Some(candidate.reference()) - ); - assert_eq!( - reopened.admitted_reference(1, 1), - Some(candidate.reference()) - ); - assert_eq!( - reopened.journal.snapshot().authenticated_ingress()[0].provenance(), - IngressProvenanceV1::Relayed { peer: 2 } - ); - let retained = reopened - .retained_authenticated_envelope(candidate.reference()) - .unwrap(); - assert_eq!(retained.canonical_carrier_wire(), wire); - assert_eq!(retained.authentication_sidecar(), sidecar); - reopened.shutdown().unwrap(); - } - #[test] fn replay_rejects_a_duplicate_authenticated_slot_even_with_an_exact_trace() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 389ee9a4..d4504394 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -36,8 +36,8 @@ use crate::{ }, network::{ NetworkMessage, RbcDagApplicationPayloadResponse, RbcDagShadowCarrier, - RbcDagShadowCarrierEnvelopeResponse, RbcDagShadowCarrierResponse, - RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, + RbcDagShadowCarrierResponse, RbcDagShadowCarrierSyncRequest, + RbcDagShadowCarrierSyncResponse, }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ @@ -310,10 +310,6 @@ enum ShadowServiceMessageV1 { peer: AuthorityIndex, response: RbcDagShadowCarrierResponse, }, - CarrierEnvelopeResponse { - peer: AuthorityIndex, - response: RbcDagShadowCarrierEnvelopeResponse, - }, CarrierSyncRequest { peer: AuthorityIndex, request: RbcDagShadowCarrierSyncRequest, @@ -544,42 +540,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { .await } - #[cfg(test)] - pub(crate) fn carrier_envelope_response( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierEnvelopeResponse, - ) -> Result<(), ShadowServiceErrorV1> { - self.validate_carrier_envelope_response(&response)?; - self.send(ShadowServiceMessageV1::CarrierEnvelopeResponse { peer, response }) - } - - pub(crate) async fn carrier_envelope_response_reliably( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierEnvelopeResponse, - ) -> Result<(), ShadowServiceErrorV1> { - self.validate_carrier_envelope_response(&response)?; - self.send_reliably(ShadowServiceMessageV1::CarrierEnvelopeResponse { peer, response }) - .await - } - - fn validate_carrier_envelope_response( - &self, - response: &RbcDagShadowCarrierEnvelopeResponse, - ) -> Result<(), ShadowServiceErrorV1> { - validate_wire_size( - "carrier envelope response", - response.canonical_carrier.len(), - MAX_CARRIER_CONTENT_SIZE_V1, - )?; - validate_wire_size( - "carrier envelope response authentication sidecar", - response.authentication_sidecar.len(), - self.max_sidecar_size, - ) - } - #[cfg(test)] pub(crate) fn carrier_sync_request( &self, @@ -894,7 +854,6 @@ impl ShadowServiceMessageV1 { Self::Carrier { .. } => "carrier", Self::CarrierRequest { .. } => "carrier_request", Self::CarrierResponse { .. } => "carrier_response", - Self::CarrierEnvelopeResponse { .. } => "carrier_envelope_response", Self::CarrierSyncRequest { .. } => "carrier_sync_request", Self::CarrierSyncResponsesChanged => "carrier_sync_responses_changed", Self::ApplicationPayloadRequest { .. } => "application_payload_request", @@ -1045,7 +1004,6 @@ pub(crate) enum ShadowServiceErrorV1 { reference: BlockReference, }, InvalidHeartbeatInterval, - InvalidConsensusTimeout, SyncRequestForForeignAuthor { expected: AuthorityIndex, actual: AuthorityIndex, @@ -1157,9 +1115,6 @@ impl fmt::Display for ShadowServiceErrorV1 { Self::InvalidHeartbeatInterval => formatter.write_str( "Starfish-RBC-DAG autonomous heartbeat interval must be nonzero", ), - Self::InvalidConsensusTimeout => formatter.write_str( - "Starfish-RBC-DAG logical consensus timeout must be nonzero", - ), Self::SyncRequestForForeignAuthor { expected, actual } => write!( formatter, "shadow carrier sync request asked authority {expected} to serve authority {actual}" @@ -1222,7 +1177,6 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::DirectMirror, - None, wal_sync_policy, None, None, @@ -1256,7 +1210,6 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::DirectMirror, - None, wal_sync_policy, Some(metrics), None, @@ -1293,7 +1246,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - None, wal_sync_policy, None, None, @@ -1331,7 +1283,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - None, wal_sync_policy, Some(metrics), None, @@ -1373,7 +1324,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_paused_with_metric authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - None, wal_sync_policy, Some(metrics), None, @@ -1394,7 +1344,6 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, heartbeat_interval: Duration, - consensus_timeout: Duration, wal_sync_policy: ShadowWalSyncPolicyV1, metrics: Arc, recovery_cursor: Option, @@ -1410,9 +1359,6 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 if heartbeat_interval.is_zero() { return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); } - if consensus_timeout.is_zero() { - return Err(ShadowServiceErrorV1::InvalidConsensusTimeout); - } start_starfish_rbc_dag_shadow_service_with_mode_v1( path, committee, @@ -1421,7 +1367,6 @@ pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1 authorizer, recovered_local_headers, ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - Some(consensus_timeout), wal_sync_policy, Some(metrics), recovery_cursor, @@ -1503,7 +1448,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, mode: ShadowServiceModeV1, - consensus_timeout: Option, wal_sync_policy: ShadowWalSyncPolicyV1, metrics: Option>, recovery_cursor: Option, @@ -1517,9 +1461,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( ShadowServiceErrorV1, > { let committee_size = committee.committee().len(); - let consensus_timeout = consensus_timeout - .or_else(|| mode.heartbeat_interval()) - .unwrap_or_default(); let input_capacity = shadow_input_capacity(committee_size, mode)?; let max_sidecar_size = authentication_sidecar_size(context.authentication_scheme(), committee_size); @@ -1962,7 +1903,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( sync_max_desired_responses: 0, awaiting_application_submission: false, consensus_pacemaker, - consensus_timeout, normal_carrier_min_spacing: mode .heartbeat_interval() .and_then(|interval| interval.checked_div(SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1)) @@ -2236,10 +2176,6 @@ struct ShadowServiceStateV1 { /// control heartbeat. Maintenance/heartbeat messages bound the wait. awaiting_application_submission: bool, consensus_pacemaker: ConsensusPacemakerV1, - /// Logical C2 fallback deadline. This is independent of the physical - /// heartbeat so experiments can vary consensus permission without - /// changing proactive carrier push cadence. - consensus_timeout: Duration, normal_carrier_min_spacing: Duration, normal_carrier_next_allowed_at: Option, normal_carrier_requested: bool, @@ -3145,10 +3081,9 @@ impl ShadowServiceStateV1 { if !self.clock_active { return false; } - if !self.mode.is_autonomous() { + let Some(leader_timeout) = self.mode.heartbeat_interval() else { return false; - } - let leader_timeout = self.consensus_timeout; + }; let slot = self.core.next_local_consensus_round(); let a1_ready = slot == 1 || self @@ -3333,9 +3268,6 @@ impl ShadowServiceStateV1 { .try_into() .unwrap_or(TimestampNs::MAX); let before = self.core.wal_counts(); - let consensus_slot = self.core.next_local_consensus_round(); - let c1_ready = self.core.local_consensus_vertex_c1_ready(); - let c3_ready = self.core.has_projected_consensus_quorum(consensus_slot); let application_round = self.pending_local.keys().next().copied(); let application = application_round.and_then(|round| self.pending_local.remove(&round)); let result = match &application { @@ -3350,8 +3282,6 @@ impl ShadowServiceStateV1 { }; match result { Ok((envelope, effects)) => { - let fixed_consensus_vertex = - self.core.next_local_consensus_round() > consensus_slot; self.record_carrier_created(Instant::now()); let initial_application_payload = application .as_ref() @@ -3387,36 +3317,6 @@ impl ShadowServiceStateV1 { }, outcome: "accepted", }); - let consensus_vertex_outcome = if !fixed_consensus_vertex { - "omitted" - } else if consensus_slot == 1 { - "bootstrap" - } else if c1_ready { - "c1" - } else if c3_ready { - "c3" - } else if allow_no_vote { - "c2" - } else { - "unexpected" - }; - self.emit(ShadowServiceEventV1::Input { - kind: "consensus_vertex", - outcome: consensus_vertex_outcome, - }); - // Keep the existing aggregate stable while exposing whether - // an available logical vertex was fixed on an application - // carrier or spent on a control/phase carrier. This is an - // event-local benchmark diagnostic and does not affect the - // carrier, journal, or consensus bytes. - self.emit(ShadowServiceEventV1::Input { - kind: if application_round.is_some() { - "application_consensus_vertex" - } else { - "control_consensus_vertex" - }, - outcome: consensus_vertex_outcome, - }); self.report_wal_delta(before); if let Some(reference) = assigned_application { self.awaiting_application_submission = true; @@ -4327,12 +4227,6 @@ fn run_shadow_service( } state.report_wal_delta(before); state.process_effects(outcome.effects().to_vec()); - // A coalesced producer notification may sit behind - // this ingress in the bounded actor FIFO even though - // its exact application is already present in the - // shared desired map. Reconcile it before phase work - // consumes the carrier round that this ingress opens. - state.reconcile_local_applications(); state.retry_pending_local(); if future_ignored { state.flush_carrier_sync_requests(true); @@ -4362,20 +4256,7 @@ fn run_shadow_service( state.reject(Some(peer), error); continue; } - if let Some(envelope) = state.core.retained_authenticated_envelope(reference) { - state.emit(ShadowServiceEventV1::Network { - recipient: peer, - message: NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( - RbcDagShadowCarrierEnvelopeResponse { - reference, - canonical_carrier: envelope.canonical_carrier_wire().to_vec(), - authentication_sidecar: envelope.authentication_sidecar().to_vec(), - }, - ), - }); - } else if let Some(canonical_carrier) = - state.core.retained_candidate_wire(reference) - { + if let Some(canonical_carrier) = state.core.retained_candidate_wire(reference) { state.emit(ShadowServiceEventV1::Network { recipient: peer, message: NetworkMessage::RbcDagShadowCarrierResponse( @@ -4459,93 +4340,6 @@ fn run_shadow_service( } } } - ShadowServiceMessageV1::CarrierEnvelopeResponse { peer, response } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - if state - .core - .authenticated_reference(response.reference.authority, response.reference.round) - == Some(response.reference) - { - state.emit(ShadowServiceEventV1::Input { - kind: "recovery", - outcome: "ignored_already_authenticated", - }); - continue; - } - let Some(holders) = state.pending_recovery.get(&response.reference) else { - state.reject( - Some(peer), - ShadowServiceErrorV1::UnexpectedResponse(response.reference), - ); - continue; - }; - if !holders.contains(&peer) { - state.reject( - Some(peer), - ShadowServiceErrorV1::ResponseFromNonHolder { - peer, - reference: response.reference, - }, - ); - continue; - } - let before = state.core.wal_counts(); - match state.core.recover_or_admit_from_peer( - response.reference, - &response.canonical_carrier, - &response.authentication_sidecar, - peer, - ) { - Ok(outcome) => { - state.pending_recovery.remove(&response.reference); - state - .recovery_last_attempt - .retain(|(target, _), _| *target != response.reference); - state.emit(ShadowServiceEventV1::Input { - kind: "recovery", - outcome: match outcome.disposition() { - ShadowIngressDispositionV1::Authenticated => { - "accepted_authenticated" - } - ShadowIngressDispositionV1::CandidateRetained => { - "accepted_content_only" - } - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { - "ignored_duplicate" - } - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => { - "future_ignored" - } - }, - }); - if let Err(error) = state.observe_carrier_application( - peer, - &response.canonical_carrier, - None, - outcome.disposition(), - ) { - state.reject(Some(peer), error); - } - state.report_wal_delta(before); - state.process_effects(outcome.effects().to_vec()); - state.retry_pending_local(); - } - Err(error) => { - state.emit(ShadowServiceEventV1::Input { - kind: "recovery", - outcome: "rejected", - }); - if is_fatal_core_error(&error) { - state.mark_fatal(error); - } else { - state.reject(Some(peer), error); - } - } - } - } ShadowServiceMessageV1::CarrierSyncRequest { peer, request } => { if let Err(error) = state.validate_peer(peer) { state.reject(Some(peer), error); @@ -5424,14 +5218,11 @@ mod tests { ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), ) .unwrap(); - let heartbeat_interval = Duration::from_secs(60); - let consensus_timeout = Duration::from_millis(20); + let leader_timeout = Duration::from_millis(20); let (mut state, mut event_rx, mut message_rx, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), heartbeat_interval); - state.consensus_timeout = consensus_timeout; + standalone_autonomous_state(core, harness.committee.clone(), leader_timeout); state.refresh_consensus_pacemaker(); - assert_eq!(state.mode.heartbeat_interval(), Some(heartbeat_interval)); assert_eq!(state.core.local_carrier_round(), 1); let (generation, slot) = match timeout(Duration::from_secs(1), message_rx.recv()) .await @@ -5759,7 +5550,6 @@ mod tests { ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), Vec::new(), ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - None, ShadowWalSyncPolicyV1::EveryBatch, None, None, @@ -5856,7 +5646,6 @@ mod tests { sync_max_desired_responses: 0, awaiting_application_submission: false, consensus_pacemaker: ConsensusPacemakerV1::new(slot), - consensus_timeout: leader_timeout, // Existing state-level tests invoke creation synchronously and do // not run the service deadline task. Keep their historical // immediate behavior unless a pacer test overrides this field. @@ -6049,12 +5838,6 @@ mod tests { response, } } - NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( - response, - ) => ShadowServiceMessageV1::CarrierEnvelopeResponse { - peer: sender as AuthorityIndex, - response, - }, NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { *sync_requests = sync_requests.saturating_add(1); ShadowServiceMessageV1::CarrierSyncRequest { @@ -6213,14 +5996,6 @@ mod tests { [recipient] .carrier_response(sender as AuthorityIndex, response) .unwrap(), - NetworkMessage::RbcDagShadowCarrierEnvelopeResponse( - response, - ) => handles[recipient] - .carrier_envelope_response( - sender as AuthorityIndex, - response, - ) - .unwrap(), NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { handles[recipient] .carrier_sync_request( @@ -8345,43 +8120,6 @@ mod tests { stop(receiver, receiver_events, receiver_task).await; } - #[tokio::test] - async fn relayed_authenticated_candidate_serves_its_exact_persisted_envelope() { - let harness = Harness::new(); - let target = round_one_candidate(0, &harness.committee, 0x22); - let envelope = harness.envelope(&target, 0); - let (holder, mut events, task) = harness.start(1, Vec::new()); - wait_ready(&mut events).await; - holder.carrier(0, envelope.clone()).unwrap(); - loop { - if let ShadowServiceEventV1::Input { - kind: "carrier", - outcome: "authenticated", - } = next_event(&mut events).await - { - break; - } - } - - holder.carrier_request(2, target.reference()).unwrap(); - loop { - if let ShadowServiceEventV1::Network { - recipient: 2, - message: NetworkMessage::RbcDagShadowCarrierEnvelopeResponse(response), - } = next_event(&mut events).await - { - assert_eq!(response.reference, target.reference()); - assert_eq!(response.canonical_carrier, envelope.canonical_carrier); - assert_eq!( - response.authentication_sidecar, - envelope.authentication_sidecar - ); - break; - } - } - stop(holder, events, task).await; - } - #[tokio::test] async fn poisoned_application_payload_waits_for_exact_delivery() { let harness = Harness::new(); @@ -8515,93 +8253,6 @@ mod tests { stop(handle, events, task).await; } - #[tokio::test] - async fn coalesced_application_wins_the_round_opened_by_carrier_ingress() { - let harness = Harness::new(); - let (handle, mut events, task) = - harness.start_autonomous_with_interval(0, Duration::from_millis(600)); - wait_ready(&mut events).await; - let (carrier_tx, mut carrier_rx) = mpsc::unbounded_channel(); - let event_task = tokio::spawn(async move { - while let Some(event) = events.recv().await { - match event { - ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrier(envelope), - } => carrier_tx.send(envelope).unwrap(), - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("coalesced application test rejected input: {error}") - } - _ => {} - } - } - }); - - handle.local_header(&direct_header(0, 1, 0x71)).unwrap(); - let first = timeout(EVENT_TIMEOUT, carrier_rx.recv()) - .await - .expect("first local carrier timed out") - .expect("carrier collector stopped"); - let first = CandidateCarrierV1::decode_wire_with_committee( - &first.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - assert_eq!(first.header().carrier_round(), 1); - tokio::time::sleep(Duration::from_millis(40)).await; - - let first_peer = round_one_candidate(1, &harness.committee, 0x72); - handle.carrier(1, harness.envelope(&first_peer, 1)).unwrap(); - let inspection = handle.inspect_carrier_sync().await.unwrap(); - assert_eq!(inspection.open_round, 1); - - // Model a full/coalesced notification queue: the producer has - // published the exact desired application, but its wake is not ahead - // of the final quorum carrier in the actor FIFO. - let application = RbcCanonicalHeader::try_new( - 0, - 2, - (0..N as AuthorityIndex) - .map(|author| BlockReference::new_test(author, 1)) - .collect(), - Vec::new(), - 2_073, - TransactionsCommitment::from_bytes([0x73; 32]), - ) - .unwrap(); - handle.desired_local_applications.lock().insert( - application.reference().round, - ShadowLocalCarrierV1::from_direct_header(&application), - ); - let quorum_peer = round_one_candidate(2, &harness.committee, 0x74); - handle - .carrier(2, harness.envelope(&quorum_peer, 2)) - .unwrap(); - let inspection = handle.inspect_carrier_sync().await.unwrap(); - assert_eq!(inspection.open_round, 2); - - let second = loop { - let envelope = timeout(EVENT_TIMEOUT, carrier_rx.recv()) - .await - .expect("second local carrier timed out") - .expect("carrier collector stopped"); - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &envelope.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - if candidate.header().carrier_round() == 2 { - break candidate; - } - }; - assert_eq!(second.header().application_header(), Some(&application)); - handle.shutdown().await.unwrap(); - task.await.unwrap(); - event_task.await.unwrap(); - } - #[tokio::test] async fn recovery_binds_holder_and_reference_then_compares_only_paired_slot_once() { let harness = Harness::new(); @@ -8758,83 +8409,6 @@ mod tests { stop(handle, events, task).await; } - #[tokio::test] - async fn vector_bearing_recovery_authenticates_relay_or_falls_back_without_blame() { - for poisoned_receiver_entry in [false, true] { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start(3, Vec::new()); - wait_ready(&mut events).await; - handle.peer_connected(0).unwrap(); - handle.peer_connected(1).unwrap(); - let target = round_one_candidate(2, &harness.committee, 0x63); - for sender in [0, 1] { - let outer = phase_carrier( - sender, - RbcPhaseStatementV1::Ready { - target: target.reference(), - }, - &harness.committee, - ); - handle - .carrier(sender, harness.envelope(&outer, sender)) - .unwrap(); - } - let holder = loop { - if let ShadowServiceEventV1::Network { - recipient, - message: NetworkMessage::RbcDagShadowCarrierRequest(reference), - } = next_event(&mut events).await - { - assert_eq!(reference, target.reference()); - break recipient; - } - }; - assert!(holder == 0 || holder == 1); - - let envelope = harness.envelope(&target, 2); - let mut authentication_sidecar = envelope.authentication_sidecar; - if poisoned_receiver_entry { - authentication_sidecar[3 + 3 * MAC_TAG_SIZE] ^= 1; - } - handle - .carrier_envelope_response( - holder, - RbcDagShadowCarrierEnvelopeResponse { - reference: target.reference(), - canonical_carrier: envelope.canonical_carrier, - authentication_sidecar, - }, - ) - .unwrap(); - - let expected_outcome = if poisoned_receiver_entry { - "accepted_content_only" - } else { - "accepted_authenticated" - }; - let mut accepted = false; - let mut delivered = false; - while !accepted || !delivered { - match next_event(&mut events).await { - ShadowServiceEventV1::Input { - kind: "recovery", - outcome, - } if outcome == expected_outcome => accepted = true, - ShadowServiceEventV1::Delivered(identity) => { - assert_eq!(identity.author, 2); - assert_eq!(identity.round, 1); - delivered = true; - } - ShadowServiceEventV1::Rejected { peer, error } => { - panic!("vector-bearing recovery assigned blame to {peer:?}: {error}") - } - _ => {} - } - } - stop(handle, events, task).await; - } - } - #[test] fn verified_payload_callback_coalesces_when_notification_queue_is_full() { let harness = Harness::new(); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 49eebc22..2db71d97 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -95,25 +95,6 @@ impl Validator { "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" )); } - if public_config - .parameters - .starfish_rbc_dag_consensus_timeout - .is_some_and(|timeout| timeout.is_zero()) - { - return Err(eyre!( - "Starfish-RBC-DAG logical consensus timeout must be nonzero" - )); - } - if public_config - .parameters - .starfish_rbc_dag_consensus_timeout - .is_some() - && !public_config.parameters.starfish_rbc_dag_autonomous_clock - { - return Err(eyre!( - "Starfish-RBC-DAG logical consensus timeout requires the autonomous clock" - )); - } if start_options.rbc_dag_clock_start_paused && (!public_config.parameters.starfish_rbc_dag_autonomous_clock || !public_config @@ -545,38 +526,6 @@ mod smoke_tests { })); } - #[tokio::test] - async fn logical_consensus_timeout_requires_autonomous_clock() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_consensus_timeout = - Some(Duration::from_millis(250)); - public_config - .parameters - .refresh_starfish_rbc_protocol_instance(); - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish-rbc".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("logical consensus timeout requires the autonomous clock") - })); - } - #[tokio::test] async fn buffered_shadow_wal_requires_shadow_mode() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 96c9c0f2..24ccf7ac 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -211,10 +211,6 @@ enum Operation { /// benchmark runs. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, - /// Override only the autonomous RBC-DAG logical C2 fallback timeout. - /// The physical carrier heartbeat remains on `leader_timeout`. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_consensus_timeout_ms: Option, /// Benchmark-only: write ordered shadow-WAL frames but force them to /// stable storage only at clean shutdown. This run is not crash-safe. #[clap(long, default_value_t = false)] @@ -338,7 +334,6 @@ async fn main() -> Result<()> { starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_single_dag_echo_qc_fast_path, - starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, duration_secs, port_offset, @@ -357,8 +352,6 @@ async fn main() -> Result<()> { starfish_rbc_dag_embedded_rbc_authority; node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = starfish_rbc_single_dag_echo_qc_fast_path; - node_parameters.starfish_rbc_dag_consensus_timeout = - starfish_rbc_dag_consensus_timeout_ms.map(Duration::from_millis); node_parameters.starfish_rbc_dag_shadow_buffered_wal = starfish_rbc_dag_shadow_buffered_wal; if is_starfish_rbc_selection(&consensus_protocol) { @@ -592,13 +585,6 @@ async fn local_benchmark( "Carrier idle timeout: {} ms (shared Starfish leader pacemaker)", node_parameters.leader_timeout.as_millis() ); - println!( - "Logical C2 timeout: {} ms", - node_parameters - .starfish_rbc_dag_consensus_timeout - .unwrap_or(node_parameters.leader_timeout) - .as_millis() - ); } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( @@ -1544,8 +1530,6 @@ mod tests { "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-single-dag-echo-qc-fast-path", - "--starfish-rbc-dag-consensus-timeout-ms", - "250", "--starfish-rbc-dag-shadow-buffered-wal", "--port-offset", "2500", @@ -1559,7 +1543,6 @@ mod tests { starfish_rbc_dag_autonomous_clock, starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_single_dag_echo_qc_fast_path, - starfish_rbc_dag_consensus_timeout_ms, starfish_rbc_dag_shadow_buffered_wal, port_offset, .. @@ -1573,7 +1556,6 @@ mod tests { assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_dag_embedded_rbc_authority); assert!(starfish_rbc_single_dag_echo_qc_fast_path); - assert_eq!(starfish_rbc_dag_consensus_timeout_ms, Some(250)); assert!(starfish_rbc_dag_shadow_buffered_wal); assert_eq!(port_offset, 2500); } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 63fefb13..cf5bcce5 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -55,9 +55,6 @@ heartbeats use the resolved Starfish leader timeout (600 ms for Push/Starfish-RB application carriers and encodable phase follow-ups are event-driven. The V4 authoritative mode promotes the proved `O`-ECHO predicate to delivery authority; `Q` READY remains a distinct slower certification fact rather than a prerequisite for fast projection. -The research harness may override the logical C2 fallback timeout independently for controlled -latency experiments. This does not change the physical heartbeat, carrier pacing, C1/C3 rules, or -wire format; absent an override, C2 continues to use the resolved Starfish leader timeout. Standalone authority is an end-to-end boundary. The direct Starfish-RBC actor is absent, generic block dissemination and legacy pull messages are rejected or ignored, and the legacy Starfish @@ -364,16 +361,12 @@ may attach different vectors to the same content reference, including a vector w one recipient and garbage for another. Correctness therefore depends only on the local entry and on the embedded four-phase protocol, never on agreement about the vector bytes. -Each node persists one exact vector variant with an authenticated carrier for restart and relay: -the locally generated sidecar for its own carrier, otherwise the first inbound variant whose local -entry verifies. The implementation never replaces that variant by arrival provenance and never -merges entries from different vectors. Exact carrier recovery after phase evidence prefers a new, -appended envelope-response message carrying the holder's persisted variant. If the requester's -local entry verifies, the response follows the ordinary relayed-ingress predicate and may create -authenticated admission, ECHO, and fast-clock stake. If the entry is absent, malformed, or invalid, -the same canonical bytes retain the legacy content-only authority and can still unblock phase -progress, authoritative delivery, and READY certification. The frozen content-only response remains -accepted for compatibility. A failed MAC check assigns no blame to either carrier author or holder. +Each node persists one exact vector variant with its carrier for restart and relay. The preference +order is locally generated, directly author-received, then first relayed variant with a valid local +entry. The implementation does not merge unverified entries from different vectors. Exact carrier +recovery after phase evidence may return canonical content without a vector; that recovery can +unblock phase progress, authoritative delivery, and READY certification, but it does not create +authenticated carrier admission or fast-clock stake. Public-signature modes use the same context-bound carrier statement without a recipient field and the same embedded RBC/consensus logic. They exist for controlled performance comparison, not as @@ -560,14 +553,12 @@ union of phase senders as candidate holders, and request the target from those h content is accepted only when canonical decoding recomputes the requested `BlockReference` and the context/committee checks succeed. Retention precedes any new local phase lock. -Recovery request/response is out-of-band byte transfer, not quorum testimony or a new phase. A -response can satisfy only an already allocated evidence obligation and cannot create one. A -vector-bearing response additionally grants ordinary carrier admission only when the exact -receiver-specific authenticator verifies under the same committee/context predicate as proactive -relayed ingress; otherwise it is content-only. The prototype retries recorded holders after GST. -Its separate carrier catch-up mechanism requests one exact `(author, round)` at a time and serves -only retained locally authored outbound bytes; it does not transfer ranges, certificates, -checkpoints, committed observer history, or arbitrary late state. +Recovery request/response is out-of-band byte transfer, not quorum testimony, admission, or a new +phase. A valid response can satisfy an already allocated evidence obligation but cannot create one. +The prototype retries recorded holders after GST. Its separate carrier catch-up mechanism requests +one exact `(author, round)` at a time and serves only retained locally authored outbound bytes; it +does not transfer ranges, certificates, checkpoints, committed observer history, or arbitrary late +state. ### 8.3 Batching and fairness @@ -626,8 +617,7 @@ an authoritative optimistic delivery: - **C1:** create at `c` after the eligible leader at `c - 1` is present and the eligible projection contains either `Q` votes for an exact leader value or a valid explicit direct-skip pattern for the leader slot at `c - 2`; -- **C2:** create after the logical consensus timeout, which defaults to the resolved Starfish - leader timeout; or +- **C2:** create after the consensus leader timeout; or - **C3:** catch up and create after observing eligible distinct-author stake `Q` already at `c`. The strong-parent set chosen under C1 must itself contain the immutable L2 witness: the exact `Q` @@ -967,13 +957,7 @@ The executable model and composed runtime tests should cover at minimum: - `M` ECHO to VOTE, `C` ECHO-or-VOTE to ACK, `C` ACK to READY, `O` authoritative delivery, and independent `Q`-READY certification; - evidence-before-content recovery from phase holders, including VOTE/ACK without local admission; -- vector-bearing phase-holder recovery that grants normal relayed admission only for a valid local - authenticator entry, falls back without blame for poisoned variants, preserves the frozen - content-only response, and replays the exact relayed provenance and sidecar after restart; - zero application load with heartbeat-only RBC completion; -- independent logical-C2 timeout scheduling without changing the physical heartbeat, plus - coalesced producer notification ordering in which an already-published application wins a newly - opened carrier round before queued phase-only work; - two-round admission, 64-round authenticated retention, and future carriers that cannot jump the local sequential clock; - `f` permanently missing weak parents without blocking honest carrier or consensus progress; @@ -1018,16 +1002,11 @@ The first fair benchmark matrix includes: - Sailfish++ as a certified signature-free comparison. Hold committee, load, transaction size, topology, latency injection, dissemination fanout, duration, -timeouts, and build constant. Report carrier, vector, ECHO, VOTE, ACK, READY, content-only recovery, -vector-bearing recovery, payload, and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery +timeouts, and build constant. Report carrier, vector, ECHO, VOTE, ACK, READY, recovery, payload, +and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained state. -The benchmark also separates logical-vertex outcomes by enclosing carrier kind. Application and -control/phase carriers each report bootstrap, C1, C2, C3, or omitted without adding per-slot metric -labels. This event-local diagnostic distinguishes pre-inclusion scheduling delay from later -projection/decision latency without changing carrier, journal, or wire bytes. - Batching can reduce the number of separately scheduled RBC control messages, but it does not remove their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout @@ -1118,9 +1097,8 @@ The following production choices remain unresolved and must be proved or measure admission lookahead `2`, retains at most `64` future rounds for temporarily descheduled peers, and discards farther unsolicited carriers before admission/retention; these are benchmark resource parameters rather than protocol safety constants); -- whether the logical C2 timeout needs a separately proved adaptive low-load rule; the prototype - exposes an experimental override but intentionally does not introduce a second physical - heartbeat timeout; +- whether the shared Starfish leader-timeout policy needs a separately proved adaptive low-load + rule; the prototype intentionally does not introduce a second heartbeat timeout; - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and From b341d452fabee0c08896fd7e1402b61a6fa4a7b5 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:38:30 +0200 Subject: [PATCH 53/62] Revert "Implement standalone Starfish RBC-DAG testbed" This reverts commit 63d0bc23b7c17c0652d2ba7b7d9d2ff47319e589. --- Cargo.lock | 1 - README.md | 65 +- crates/orchestrator/src/measurements.rs | 63 +- crates/starfish-core/src/block_handler.rs | 57 +- crates/starfish-core/src/config.rs | 10 +- crates/starfish-core/src/core.rs | 740 +-- .../starfish-core/src/core_thread/spawned.rs | 131 +- crates/starfish-core/src/dag_state.rs | 37 - crates/starfish-core/src/metrics.rs | 1478 +---- crates/starfish-core/src/net_sync.rs | 2998 +--------- crates/starfish-core/src/network.rs | 1283 +---- crates/starfish-core/src/rocks_store.rs | 208 +- .../src/starfish_rbc_dag/journal.rs | 648 +-- .../starfish-core/src/starfish_rbc_dag/mod.rs | 96 +- .../src/starfish_rbc_dag/model.rs | 1065 +--- .../src/starfish_rbc_dag/projection.rs | 1319 +---- .../src/starfish_rbc_dag_shadow.rs | 2407 +------- .../src/starfish_rbc_dag_shadow_service.rs | 5075 +---------------- crates/starfish-core/src/stat.rs | 10 - crates/starfish-core/src/store.rs | 290 +- crates/starfish-core/src/syncer.rs | 631 +- crates/starfish-core/src/tidehunter_store.rs | 201 +- .../src/transactions_generator.rs | 448 +- crates/starfish-core/src/validator.rs | 123 +- crates/starfish/Cargo.toml | 1 - crates/starfish/src/main.rs | 714 +-- docs/starfish-rbc-dag-protocol.md | 737 ++- 27 files changed, 1790 insertions(+), 19046 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d586ec61..fc80648f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,7 +3584,6 @@ dependencies = [ "clap", "color-eyre", "eyre", - "futures", "prettytable-rs", "starfish-core", "tokio", diff --git a/README.md b/README.md index b39e91ba..855c189c 100644 --- a/README.md +++ b/README.md @@ -48,44 +48,30 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is the standalone follow-up that carries authentication, reliable-broadcast -control, application headers, and logical Starfish vertices in one optimistic carrier DAG. Run the -comparison plane with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; add -`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to make that carrier -plane the sole reliable-broadcast, consensus, ordering, and output authority. In this mode the direct -Starfish-RBC service is not started: direct INIT/phase/header-recovery messages, generic block -batches, and legacy parent/transaction pulls cannot certify or order an application. Application -bytes travel with the carrier envelope when available, or through the dedicated RBC-DAG payload -request/response path, and are accepted only after commitment verification. - -The implemented MAC-vector RBC uses four embedded phases. For target-author stake `a` and total -stake `W`, ECHO, VOTE, and ACK exclude the target author; weighted thresholds `M`, `C`, and `O` -drive VOTE, ACK/READY convergence, and authoritative optimistic delivery. Reaching `O` ECHO stake -is sufficient for the fast delivery latch, while `Q = W - floor((W - 1) / 3)` READY stake records a -separate slower certification latch. If `a > floor((W - 1) / 3)`, the fault model makes the author -honest and receiver-authenticated exact content can take the fast latch directly. All four phases -use per-sender and local slot-global locks, and threshold evidence without content triggers exact -carrier recovery before a local follow-up is exposed. The full definitions and safety boundary are -in the [protocol design](docs/starfish-rbc-dag-protocol.md). - -Autonomous carriers embed durably locked consensus vertices with quorum strong parents, explicit -Vote/NoVote choices, and exact delivery frontiers. Authoritative delivery alone is not application -data availability: projection and output wait until Core has materialized the concrete application -block and its committed payload is available. Each committed anchor is componentwise joined into a -cumulative committed frontier; only the new exact, data-available prefix delta is released, and -the legacy Starfish committer is disabled. The prototype admits at most two carrier rounds ahead, -retains canonical unsolicited carrier content up to 64 rounds ahead, and offers rate-limited -single-slot exact synchronization rather than checkpoint or proof-safe late-node state transfer. -Its current authoritative journal uses the V4 autonomous WAL namespace with `SRD5` raw records so -older traces cannot be reinterpreted under the optimistic-delivery rules. - -The default WAL syncs every transition. `--starfish-rbc-dag-shadow-buffered-wal` preserves ordered -frames but syncs only on clean shutdown and is not crash-safe. Actor replay covers the state -explicitly documented in the protocol design; full validator crash recovery, bounded checkpoint -transfer, and proof-safe state retirement are not claimed. Shadow traffic shares the validator's -network socket and bandwidth, and deployment requires a homogeneous new-binary committee. Idle -carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for Starfish-RBC by default); -application and encodable phase carriers are emitted immediately. +**Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG +while keeping certified Starfish consensus and ordering in a separate logical projection. Its +canonical types, deterministic models, crash journal, comparison shadow, autonomous carrier clock, +authoritative embedded-RBC path, and certified logical consensus projection are implemented. Run +the comparison shadow with +`--consensus starfish-rbc --starfish-rbc-dag-shadow`; add +`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact +application headers in version-two carriers and make embedded ECHO/READY/delivery their sole +certification authority. Direct INIT remains payload transport, but direct ECHO/READY cannot clean +blocks in that mode. Committed projected anchors now release deterministic carrier-frontier deltas, +and those deltas are the sole application ordering/output authority; the legacy Starfish committer +is disabled. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for +Starfish-RBC by default); application and encodable phase carriers are emitted immediately. + +Autonomous carriers now embed durably locked consensus vertices with quorum strong parents, +explicit Vote/NoVote choices, and exact delivery frontiers. Only RBC-delivered, data-available, +prefix-closed vertices enter the projection or its leader decisions. Frontier output retains exact +application references and is rebuilt from the ordered WAL on actor reopen. Full validator crash +recovery and proof-safe late-node state transfer remain outside this milestone. Shadow traffic shares the +validator's network socket and bandwidth, and deployment requires a homogeneous new-binary +committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; +`--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean +shutdown and therefore forfeits crash safety. Full validator crash recovery also remains out of +scope. See the [protocol design](docs/starfish-rbc-dag-protocol.md). For a direct-header shadow comparison, `starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded observational path was disabled or shed work and the comparison must be discarded. Healthy live @@ -99,8 +85,7 @@ supports at most 60 validators in mirror mode and 20 in autonomous mode. A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared -600 ms leader/idle-carrier timeout. These milestone rows predate the current four-phase V4 -authority model and are retained as historical measurements. +600 ms leader/idle-carrier timeout. | Profile | Verdict | TPS | Block latency | E2E latency | Outbound BW | |---|---:|---:|---:|---:|---:| diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index ad68be75..51af511c 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -16,11 +16,10 @@ use prettytable::{Table, row}; use prometheus_parse::Scrape; use serde::{Deserialize, Serialize}; use starfish_core::metrics::{ + STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR, STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR, STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, - starfish_rbc_dag_autonomous_buffered_capacity_bound, - starfish_rbc_dag_autonomous_buffered_settled_bound, }; use crate::{ @@ -753,12 +752,6 @@ impl MeasurementsCollection { .is_some_and(|measurement| measurement.scalar > minimum) } - fn latest_scalar_at_most(&self, label: &str, scraper_id: ScraperId, maximum: f64) -> bool { - self.scraper_series(label, scraper_id) - .and_then(|series| series.last()) - .is_some_and(|measurement| measurement.scalar >= 0.0 && measurement.scalar <= maximum) - } - fn scalar_gauge_increased(&self, label: &str, scraper_id: ScraperId) -> bool { self.active_window_series(label, scraper_id) .is_some_and(|series| { @@ -1186,11 +1179,8 @@ impl MeasurementsCollection { }; let autonomous_phase_backlog_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let committee_size = i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX); - let autonomous_buffered_capacity_bound = - starfish_rbc_dag_autonomous_buffered_capacity_bound(committee_size); - let autonomous_buffered_settled_bound = - starfish_rbc_dag_autonomous_buffered_settled_bound(committee_size); + let autonomous_buffered_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR + .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); let every_autonomous_scraper_has_progress = shadow_autonomous_clock_valid_scrapers .iter() .all(|scraper_id| { @@ -1257,12 +1247,7 @@ impl MeasurementsCollection { && self.gauge_always_at_most( "starfish_rbc_dag_shadow_buffered_authenticated", *scraper_id, - autonomous_buffered_capacity_bound as f64, - ) - && self.latest_scalar_at_most( - "starfish_rbc_dag_shadow_buffered_authenticated", - *scraper_id, - autonomous_buffered_settled_bound as f64, + autonomous_buffered_bound as f64, ) }); let shadow_autonomous_clock_valid = shadow_autonomous_clock_enabled @@ -2011,13 +1996,10 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 #[test] fn autonomous_clock_has_a_distinct_sticky_summary() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); - // Catch-up may transiently use most of the 62 retained slots per - // remote author. That is safe as long as the final healthy tail - // settles to the tighter round-skew-derived bound. - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 1, 2, 7, 50, 5); - add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 8, 1, 2, 6, 60, 6); + add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 1, 2, 7, 1, 5); + add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 8, 1, 2, 6, 1, 6); add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 3, 2, 7, 2, 10); - add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 10, 4, 2, 6, 2, 11); + add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 10, 4, 2, 6, 3, 11); let summary = collection.benchmark_run_summary(); assert!(!summary.shadow_comparison_enabled); @@ -2032,7 +2014,7 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert_eq!(summary.shadow_autonomous_clock_admitted_stake_min, 6); assert_eq!( summary.shadow_autonomous_clock_buffered_authenticated_total, - 4 + 5 ); assert_eq!(summary.shadow_wal_durable_records, 21); @@ -2044,35 +2026,6 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert!(!summary.shadow_autonomous_clock_valid); } - #[test] - fn autonomous_clock_buffer_gate_separates_capacity_from_settled_tail() { - let mut capacity_overflow = - MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); - add_autonomous_clock_snapshot(&mut capacity_overflow, 0, 1.0, 8, 0, 2, 7, 63, 5); - add_autonomous_clock_snapshot(&mut capacity_overflow, 1, 1.0, 8, 0, 2, 7, 0, 5); - add_autonomous_clock_snapshot(&mut capacity_overflow, 0, 1.0, 12, 0, 2, 7, 2, 10); - add_autonomous_clock_snapshot(&mut capacity_overflow, 1, 1.0, 12, 0, 2, 7, 2, 10); - assert!( - !capacity_overflow - .benchmark_run_summary() - .shadow_autonomous_clock_valid, - "one remote author cannot occupy more than the 62-slot retention capacity" - ); - - let mut unsettled_tail = - MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); - add_autonomous_clock_snapshot(&mut unsettled_tail, 0, 1.0, 8, 0, 2, 7, 50, 5); - add_autonomous_clock_snapshot(&mut unsettled_tail, 1, 1.0, 8, 0, 2, 7, 50, 5); - add_autonomous_clock_snapshot(&mut unsettled_tail, 0, 1.0, 12, 0, 2, 7, 3, 10); - add_autonomous_clock_snapshot(&mut unsettled_tail, 1, 1.0, 12, 0, 2, 7, 2, 10); - assert!( - !unsettled_tail - .benchmark_run_summary() - .shadow_autonomous_clock_valid, - "a healthy final scrape must settle to two buffered slots per remote author" - ); - } - #[test] fn autonomous_clock_buffered_wal_uses_appended_progress_without_claiming_durability() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); diff --git a/crates/starfish-core/src/block_handler.rs b/crates/starfish-core/src/block_handler.rs index 28076f35..76bc4269 100644 --- a/crates/starfish-core/src/block_handler.rs +++ b/crates/starfish-core/src/block_handler.rs @@ -32,9 +32,9 @@ const REAL_BLOCK_HANDLER_TXN_SIZE: usize = 512; const REAL_BLOCK_HANDLER_TXN_GEN_STEP: usize = 32; const _: () = assert_constants(); -#[allow(dead_code, clippy::manual_is_multiple_of)] +#[allow(dead_code)] const fn assert_constants() { - if REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0 { + if !REAL_BLOCK_HANDLER_TXN_SIZE.is_multiple_of(REAL_BLOCK_HANDLER_TXN_GEN_STEP) { panic!("REAL_BLOCK_HANDLER_TXN_SIZE % REAL_BLOCK_HANDLER_TXN_GEN_STEP != 0") } } @@ -121,13 +121,13 @@ impl RealCommitHandler { } fn transaction_observer(&self, block: Data) { - // Transaction observations stay open for the bounded post-submission - // drain. No transactions exist before the coordinated start, so this - // records exactly the offered window while allowing the harness to - // distinguish committed-at-cutoff from eventual committed throughput. + // Skip every rate-feeding metric outside the active submission + // window. Late commits during warmup or wind-down would otherwise + // skew TPS, the cumulative latency distribution, and the bandwidth- + // efficiency denominator. if !self .metrics - .transaction_metrics_active + .metrics_active .load(std::sync::atomic::Ordering::Relaxed) { return; @@ -144,15 +144,6 @@ impl RealCommitHandler { .transaction_committed_latency_squared_micros .inc_by(latency.as_micros().pow(2) as u64); self.metrics.sequenced_transactions_total.inc(); - let cutoff_micros = self - .metrics - .benchmark_transaction_cutoff_micros - .load(std::sync::atomic::Ordering::Acquire); - if cutoff_micros > 0 - && self.metrics.validator_start.elapsed().as_micros() < cutoff_micros.into() - { - self.metrics.sequenced_transactions_cutoff_total.inc(); - } self.metrics .sequenced_transactions_bytes .inc_by(transaction.as_bytes().len() as u64); @@ -377,19 +368,37 @@ impl CommitObserver for RealCommitHandler { resulted_committed } - fn handle_rbc_dag_commit(&mut self, committed: &[CommittedSubDag]) { - self.record_commit_metadata(committed); - for commit in committed { - for block in &commit.blocks { - if block.round() > 0 { - self.transaction_observer(block.clone()); - } + fn handle_rbc_dag_commit( + &mut self, + dag_state: &DagState, + anchor: BlockReference, + applications: &[BlockReference], + ) -> Vec { + let blocks = applications + .iter() + .map(|reference| { + let block = dag_state + .get_storage_block(*reference) + .unwrap_or_else(|| panic!("committed RBC-DAG application {reference} missing")); + assert!( + dag_state.is_data_available(reference), + "committed RBC-DAG application {reference} is unavailable" + ); + block + }) + .collect::>(); + let commit = CommittedSubDag::new(anchor, blocks); + self.record_commit_metadata(std::iter::once(&commit)); + for block in &commit.blocks { + if block.round() > 0 { + self.transaction_observer(block.clone()); } } - self.sequenced_commit_count += committed.len(); + self.sequenced_commit_count += 1; self.metrics .commit_availability_gap .set((self.committed_count - self.sequenced_commit_count) as i64); + vec![commit] } fn recover_committed( diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 0c10344d..45721202 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -414,11 +414,7 @@ impl NodePrivateConfig { pub fn starfish_rbc_dag_autonomous_clock_wal(&self) -> PathBuf { self.storage_path - // V4 promotes the proved optimistic ECHO predicate from planning - // into authoritative RBC delivery while retaining the separate - // Q-READY certificate latch. Older traces must not be reinterpreted - // under the stronger output semantics. - .join("starfish-rbc-dag-autonomous-clock-v4.wal") + .join("starfish-rbc-dag-autonomous-clock-v1.wal") } pub fn starfish_rbc_dag_shadow_buffered_benchmark_wal(&self) -> PathBuf { @@ -428,7 +424,7 @@ impl NodePrivateConfig { pub fn starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal(&self) -> PathBuf { self.storage_path - .join("starfish-rbc-dag-autonomous-clock-buffered-benchmark-v3.wal") + .join("starfish-rbc-dag-autonomous-clock-buffered-benchmark-v1.wal") } } @@ -503,7 +499,7 @@ mod tests { private_config.starfish_rbc_dag_autonomous_clock_wal(), Path::new("benchmark") .join("storage-0") - .join("starfish-rbc-dag-autonomous-clock-v4.wal") + .join("starfish-rbc-dag-autonomous-clock-v1.wal") ); } } diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index d2970e8b..68bf47ce 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -31,10 +31,8 @@ use crate::{ encoder::ShardEncoder, metrics::{Metrics, UtilizationTimerVecExt}, runtime::timestamp_utc, - starfish_rbc_dag::ConsensusVertexReference, - starfish_rbc_dag_shadow::RbcDagFrontierRecoveryCursorV1, state::RecoveredState, - store::{RbcDagFrontierReceipt, Store}, + store::Store, types::{ AuthorityIndex, AuthoritySet, BaseTransaction, BlockAuthenticationScheme, BlockAuthorizer, BlockReference, BlsAggregateCertificate, Encoder, PartialSig, PartialSigKind, @@ -81,103 +79,6 @@ pub struct Core { /// descriptors only. Their dirty/clean DAG is no longer a consensus or /// output authority, so raw threshold-clock progress may produce them. rbc_dag_application_production: bool, - /// Latest atomically persisted authoritative carrier-frontier cursor. - /// Loaded before the shadow actor opens and advanced only after the - /// commit/receipt storage batch succeeds. - latest_rbc_dag_frontier_cursor: Option, -} - -#[derive(Debug)] -pub(crate) enum RbcDagFrontierApplyError { - StaleSequence { - current_sequence: RoundNumber, - actual_sequence: RoundNumber, - }, - SequenceGap { - expected_sequence: RoundNumber, - actual_sequence: RoundNumber, - }, - ConflictingAnchor { - output_sequence: RoundNumber, - expected: BlockReference, - actual: BlockReference, - }, - ConflictingApplications { - output_sequence: RoundNumber, - anchor: BlockReference, - expected: Vec, - actual: Vec, - }, - ReusedAnchor { - anchor: BlockReference, - previous_sequence: RoundNumber, - actual_sequence: RoundNumber, - }, - MissingApplication(BlockReference), - UnavailableApplication(BlockReference), -} - -impl fmt::Display for RbcDagFrontierApplyError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::StaleSequence { - current_sequence, - actual_sequence, - } => write!( - formatter, - "stale RBC-DAG frontier output sequence {actual_sequence}; durable sequence is {current_sequence}" - ), - Self::SequenceGap { - expected_sequence, - actual_sequence, - } => write!( - formatter, - "RBC-DAG frontier output sequence gap: expected {expected_sequence}, got {actual_sequence}" - ), - Self::ConflictingAnchor { - output_sequence, - expected, - actual, - } => write!( - formatter, - "conflicting RBC-DAG frontier anchor at output sequence {output_sequence}: durable {expected}, actual {actual}" - ), - Self::ReusedAnchor { - anchor, - previous_sequence, - actual_sequence, - } => write!( - formatter, - "RBC-DAG carrier anchor {anchor} was reused at output sequence {actual_sequence} after durable sequence {previous_sequence}" - ), - Self::ConflictingApplications { - output_sequence, - anchor, - expected, - actual, - } => write!( - formatter, - "conflicting RBC-DAG frontier applications at output sequence {output_sequence} for anchor {anchor}: durable {expected:?}, actual {actual:?}" - ), - Self::MissingApplication(reference) => { - write!( - formatter, - "committed RBC-DAG application {reference} is missing" - ) - } - Self::UnavailableApplication(reference) => write!( - formatter, - "committed RBC-DAG application {reference} is unavailable" - ), - } - } -} - -impl std::error::Error for RbcDagFrontierApplyError {} - -pub(crate) enum RbcDagFrontierApplyOutcome { - Applied(Vec), - ExactReplay, } #[derive(Debug, Clone)] @@ -206,40 +107,6 @@ impl Core { committed_leaders_count, } = recovered; - let latest_rbc_dag_frontier_receipt = store - .read_latest_rbc_dag_frontier_receipt() - .expect("Failed to read the latest RBC-DAG frontier receipt"); - let latest_rbc_dag_frontier_cursor = latest_rbc_dag_frontier_receipt.map(|receipt| { - assert!( - dag_state.consensus_protocol.is_starfish_rbc(), - "an RBC-DAG frontier receipt cannot be recovered under a non-RBC protocol" - ); - dag_state.restore_rbc_dag_committed_rounds(&receipt.committed_rounds); - let application_references = store - .get_commit(&receipt.carrier_anchor) - .expect("Failed to read the RBC-DAG frontier application commit") - .map(|commit| { - assert!( - !commit.sub_dag.is_empty(), - "a present RBC-DAG frontier application commit must not be empty; control-only frontiers are represented by absence" - ); - assert_eq!( - commit.leader, receipt.carrier_anchor, - "RBC-DAG frontier application commit must be keyed by its carrier anchor" - ); - assert_eq!( - commit.committed_rounds, receipt.committed_rounds, - "RBC-DAG frontier application commit watermarks must match its atomic receipt" - ); - commit.sub_dag - }) - .unwrap_or_default(); - RbcDagFrontierRecoveryCursorV1 { - receipt, - application_references, - } - }); - // Use genesis blocks cached in DagState (already inserted into DAG on // clean start by DagState::open()). Threshold clock is also initialized // inside DagState::open(). @@ -341,7 +208,6 @@ impl Core { committer, encoder, rbc_dag_application_production: false, - latest_rbc_dag_frontier_cursor, }; if !unprocessed_blocks.is_empty() { @@ -1750,128 +1616,29 @@ impl Core { self.flush_pending_clean_refs(); } - /// Atomically persist one authoritative RBC-DAG frontier and its optional - /// application commit. Classification against the in-memory durable - /// cursor happens before application materialization or observer effects. - pub(crate) fn handle_rbc_dag_committed_delta( - &mut self, - output_sequence: RoundNumber, - anchor: ConsensusVertexReference, - applications: &[BlockReference], - ) -> Result { + /// Persist an M7 frontier delta without feeding the obsolete Starfish + /// clean-DAG commit/proposal watermarks back into block production. + pub fn handle_rbc_dag_committed_delta(&mut self, committed: Vec) { let _timer = self .metrics .utilization_timer .utilization_timer("Core::handle_rbc_dag_committed_delta"); - - if let Some(current) = &self.latest_rbc_dag_frontier_cursor { - if output_sequence < current.receipt.output_sequence { - return Err(RbcDagFrontierApplyError::StaleSequence { - current_sequence: current.receipt.output_sequence, - actual_sequence: output_sequence, - }); - } - if output_sequence == current.receipt.output_sequence { - if anchor.carrier() == current.receipt.carrier_anchor { - if applications == current.application_references { - return Ok(RbcDagFrontierApplyOutcome::ExactReplay); - } - return Err(RbcDagFrontierApplyError::ConflictingApplications { - output_sequence, - anchor: anchor.carrier(), - expected: current.application_references.clone(), - actual: applications.to_vec(), - }); - } - return Err(RbcDagFrontierApplyError::ConflictingAnchor { - output_sequence, - expected: current.receipt.carrier_anchor, - actual: anchor.carrier(), - }); - } - if anchor.carrier() == current.receipt.carrier_anchor { - return Err(RbcDagFrontierApplyError::ReusedAnchor { - anchor: anchor.carrier(), - previous_sequence: current.receipt.output_sequence, - actual_sequence: output_sequence, - }); - } - let expected_sequence = current.receipt.output_sequence.checked_add(1).ok_or( - RbcDagFrontierApplyError::SequenceGap { - expected_sequence: RoundNumber::MAX, - actual_sequence: output_sequence, - }, - )?; - if output_sequence != expected_sequence { - return Err(RbcDagFrontierApplyError::SequenceGap { - expected_sequence, - actual_sequence: output_sequence, - }); - } - } else if output_sequence != 1 { - return Err(RbcDagFrontierApplyError::SequenceGap { - expected_sequence: 1, - actual_sequence: output_sequence, - }); + let mut commit_data = Vec::with_capacity(committed.len()); + for commit in &committed { + self.dag_state.update_last_committed_rounds(commit); + commit_data.push(CommitData::new( + commit, + self.dag_state.last_committed_rounds(), + )); } - - let blocks = applications - .iter() - .map(|reference| { - let block = self - .dag_state - .get_storage_block(*reference) - .ok_or(RbcDagFrontierApplyError::MissingApplication(*reference))?; - if !self.dag_state.is_data_available(reference) { - return Err(RbcDagFrontierApplyError::UnavailableApplication(*reference)); - } - Ok(block) - }) - .collect::, RbcDagFrontierApplyError>>()?; - let committed = CommittedSubDag::new(anchor.carrier(), blocks); - self.dag_state.update_last_committed_rounds(&committed); - let committed_rounds = self.dag_state.last_committed_rounds(); - let receipt = RbcDagFrontierReceipt { - carrier_anchor: anchor.carrier(), - output_sequence, - committed_rounds: committed_rounds.clone(), - }; - // A control-only frontier has no application CommitData, but the - // receipt still advances atomically through the same storage API. - let commit_data = (!applications.is_empty()) - .then(|| CommitData::new(&committed, committed_rounds)) - .into_iter() - .collect(); let store_start = std::time::Instant::now(); self.store - .store_commits_with_rbc_dag_receipt(commit_data, receipt.clone()) + .store_commits(commit_data) .expect("Store RBC-DAG frontier commits should not fail"); self.metrics .store_commits_latency_us .inc_by(store_start.elapsed().as_micros() as u64); self.metrics.store_commits_count.inc(); - self.latest_rbc_dag_frontier_cursor = Some(RbcDagFrontierRecoveryCursorV1 { - receipt, - application_references: applications.to_vec(), - }); - Ok(RbcDagFrontierApplyOutcome::Applied(vec![committed])) - } - - #[cfg(test)] - pub(crate) fn latest_rbc_dag_frontier_receipt(&self) -> Option { - self.latest_rbc_dag_frontier_cursor - .as_ref() - .map(|cursor| cursor.receipt.clone()) - } - - /// Return the exact runtime recovery cursor before Core is moved into its - /// dispatcher. The durable receipt intentionally remains compact; exact - /// application references are reconstructed from the atomic CommitData - /// stored under the carrier anchor. - pub(crate) fn rbc_dag_frontier_recovery_cursor( - &self, - ) -> Option { - self.latest_rbc_dag_frontier_cursor.clone() } pub(crate) fn enable_rbc_dag_application_production(&mut self) { @@ -1884,41 +1651,16 @@ impl Core { pub fn write_commits(&mut self, _commits: &[CommitData]) {} - pub fn take_recovered_committed( - &mut self, - rbc_dag_frontier_authority: bool, - ) -> (AHashSet, usize) { - let legacy_committed_blocks = self + pub fn take_recovered_committed(&mut self) -> (AHashSet, usize) { + let committed_blocks = self .recovered_committed_blocks .take() .expect("take_recovered_committed called twice"); - let legacy_committed_leaders_count = self + let committed_leaders_count = self .recovered_committed_leaders_count .take() .expect("take_recovered_committed called twice"); - if !rbc_dag_frontier_authority { - return (legacy_committed_blocks, legacy_committed_leaders_count); - } - - assert!( - self.dag_state.consensus_protocol.is_starfish_rbc(), - "embedded RBC-DAG observer recovery requires the Starfish-RBC protocol" - ); - // Carrier-keyed CommitData is intentionally not discoverable through - // DagState's Core-block scan, and the latest application CommitData - // contains only one frontier delta. Exactly-once is owned by the - // durable receipt plus the authoritative WAL; the observer needs only - // its monotone output count in this mode and never runs the legacy - // Linearizer. - let committed_frontier_count = self - .latest_rbc_dag_frontier_cursor - .as_ref() - .map(|cursor| { - usize::try_from(cursor.receipt.output_sequence) - .expect("RBC-DAG output sequence must fit usize") - }) - .unwrap_or_default(); - (AHashSet::new(), committed_frontier_count) + (committed_blocks, committed_leaders_count) } pub fn dag_state(&self) -> &DagState { @@ -1978,13 +1720,10 @@ mod tests { bls_certificate_aggregator::CertificateEvent, config::{DisseminationMode, NodePrivateConfig, StorageBackend}, crypto::{self, BlsSigner, Signer}, - dag_state::{CommitData, DagState, DataSource}, + dag_state::{DagState, DataSource}, data::Data, metrics::Metrics, - types::{ - AuthoritySet, BlockReference, BlsAggregateCertificate, Transaction, TransactionData, - VerifiedBlock, - }, + types::{AuthoritySet, BlockReference, BlsAggregateCertificate, VerifiedBlock}, }; struct NoopBlockHandler; @@ -2085,14 +1824,6 @@ mod tests { Data::new(block) } - fn rbc_application_is_materialized( - core: &Core, - reference: BlockReference, - ) -> bool { - core.dag_state().get_storage_block(reference).is_some() - && core.dag_state().is_data_available(&reference) - } - fn make_test_round_certificate( bls_signers: &[BlsSigner], round: RoundNumber, @@ -2293,441 +2024,6 @@ mod tests { ); } - #[test] - fn rbc_dag_control_frontier_cursor_reopens_and_binds_exact_empty_output() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let open = || { - DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ) - }; - let (mut core, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - NodePrivateConfig::new_for_tests(authority), - metrics.clone(), - open(), - None, - ); - let anchor = ConsensusVertexReference::new(BlockReference::new_test(2, 20), 7); - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, anchor, &[]), - Ok(RbcDagFrontierApplyOutcome::Applied(_)) - )); - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, anchor, &[]), - Ok(RbcDagFrontierApplyOutcome::ExactReplay) - )); - let unexpected = BlockReference::new_test(1, 3); - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, anchor, &[unexpected]), - Err(RbcDagFrontierApplyError::ConflictingApplications { - output_sequence: 1, - expected, - actual, - .. - }) if expected.is_empty() && actual == vec![unexpected] - )); - drop(core); - - let (mut reopened, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - NodePrivateConfig::new_for_tests(authority), - metrics.clone(), - open(), - None, - ); - let cursor = reopened - .rbc_dag_frontier_recovery_cursor() - .expect("control-only receipt must reopen as an exact cursor"); - assert_eq!(cursor.receipt.carrier_anchor, anchor.carrier()); - assert_eq!(cursor.receipt.output_sequence, 1); - assert!(cursor.application_references.is_empty()); - assert!(matches!( - reopened.handle_rbc_dag_committed_delta(1, anchor, &[]), - Ok(RbcDagFrontierApplyOutcome::ExactReplay) - )); - } - - #[test] - fn rbc_dag_application_frontier_cursor_reconstructs_exact_references() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let open = || { - DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ) - }; - let (mut core, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - NodePrivateConfig::new_for_tests(authority), - metrics.clone(), - open(), - None, - ); - let application = make_starfish_rbc_round_1_block(&committee, 1); - let application_reference = *application.reference(); - core.add_blocks(vec![(application, None)], DataSource::BlockBundleStreaming); - assert!(core.dag_state().is_data_available(&application_reference)); - let anchor = ConsensusVertexReference::new(BlockReference::new_test(3, 30), 9); - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, anchor, &[application_reference]), - Ok(RbcDagFrontierApplyOutcome::Applied(_)) - )); - drop(core); - - let (mut reopened, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - NodePrivateConfig::new_for_tests(authority), - metrics.clone(), - open(), - None, - ); - let cursor = reopened - .rbc_dag_frontier_recovery_cursor() - .expect("application frontier receipt must reopen with exact references"); - assert_eq!(cursor.application_references, vec![application_reference]); - assert!(matches!( - reopened.handle_rbc_dag_committed_delta(1, anchor, &[application_reference]), - Ok(RbcDagFrontierApplyOutcome::ExactReplay) - )); - assert!(matches!( - reopened.handle_rbc_dag_committed_delta(1, anchor, &[]), - Err(RbcDagFrontierApplyError::ConflictingApplications { .. }) - )); - } - - #[test] - fn rbc_dag_frontier_sequence_accepts_regressing_anchor_round_and_rejects_gaps() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let recovered = DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ); - let (mut core, _) = Core::open( - NoopBlockHandler, - authority, - committee, - NodePrivateConfig::new_for_tests(authority), - metrics, - recovered, - None, - ); - let later_certifier = ConsensusVertexReference::new(BlockReference::new_test(2, 50), 8); - let older_leader = ConsensusVertexReference::new(BlockReference::new_test(1, 40), 3); - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, later_certifier, &[]), - Ok(RbcDagFrontierApplyOutcome::Applied(_)) - )); - assert!(matches!( - core.handle_rbc_dag_committed_delta(3, older_leader, &[]), - Err(RbcDagFrontierApplyError::SequenceGap { - expected_sequence: 2, - actual_sequence: 3 - }) - )); - assert!(matches!( - core.handle_rbc_dag_committed_delta(2, older_leader, &[]), - Ok(RbcDagFrontierApplyOutcome::Applied(_)) - )); - let receipt = core.latest_rbc_dag_frontier_receipt().unwrap(); - assert_eq!(receipt.output_sequence, 2); - assert_eq!(receipt.carrier_anchor, older_leader.carrier()); - let (legacy_refs, observer_count) = core.take_recovered_committed(true); - assert!(legacy_refs.is_empty()); - assert_eq!(observer_count, 2); - } - - #[test] - #[should_panic(expected = "a present RBC-DAG frontier application commit must not be empty")] - fn rbc_dag_frontier_reopen_rejects_present_empty_commit_data() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let recovered = DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ); - let anchor = BlockReference::new_test(2, 7); - let receipt = RbcDagFrontierReceipt { - carrier_anchor: anchor, - output_sequence: 1, - committed_rounds: vec![0; committee.len()], - }; - recovered - .store - .store_commits_with_rbc_dag_receipt(Vec::new(), receipt.clone()) - .unwrap(); - recovered - .store - .store_commits(vec![CommitData { - leader: anchor, - sub_dag: Vec::new(), - committed_rounds: receipt.committed_rounds, - }]) - .unwrap(); - - let _ = Core::open( - NoopBlockHandler, - authority, - committee, - NodePrivateConfig::new_for_tests(authority), - metrics, - recovered, - None, - ); - } - - #[test] - fn rbc_dag_frontier_rejects_missing_or_unavailable_applications_without_advancing_receipt() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let recovered = DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ); - let private_config = NodePrivateConfig::new_for_tests(authority); - let (mut core, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - private_config, - metrics, - recovered, - None, - ); - let anchor = ConsensusVertexReference::new(BlockReference::new_test(authority, 1), 1); - let missing = BlockReference::new_test(1, 7); - - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, anchor, &[missing]), - Err(RbcDagFrontierApplyError::MissingApplication(reference)) if reference == missing - )); - assert!(core.latest_rbc_dag_frontier_receipt().is_none()); - - let transactions = vec![BaseTransaction::Share(Transaction::new(vec![9; 64]))]; - let mut encoder = Encoder::new(2, 4, 2).unwrap(); - let encoded = encoder.encode_transactions( - &transactions, - committee.info_length(), - committee.len() - committee.info_length(), - ); - let mut unavailable = VerifiedBlock::new_starfish_rbc( - 1, - 1, - committee - .authorities() - .map(|parent| BlockReference::new_test(parent, 0)) - .collect(), - Vec::new(), - 1, - Vec::new(), - Some(encoded), - ); - unavailable.preserialize(); - let unavailable = Data::new(unavailable); - let unavailable_reference = *unavailable.reference(); - let (processed, missing_parents, processed_references, _) = - core.add_headers(vec![unavailable], DataSource::BlockBundleStreamingHeader); - assert!(processed); - assert!(missing_parents.is_empty()); - assert!(processed_references.contains(&unavailable_reference)); - assert!(!core.dag_state().is_data_available(&unavailable_reference)); - - assert!(matches!( - core.handle_rbc_dag_committed_delta(1, anchor, &[unavailable_reference]), - Err(RbcDagFrontierApplyError::UnavailableApplication(reference)) - if reference == unavailable_reference - )); - assert!(core.latest_rbc_dag_frontier_receipt().is_none()); - } - - #[test] - fn buffered_payload_materializes_only_after_missing_header_parents_arrive() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let recovered = DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ); - let private_config = NodePrivateConfig::new_for_tests(authority); - let (mut core, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - private_config, - metrics, - recovered, - None, - ); - - let parents = [1, 2, 3] - .into_iter() - .map(|peer| make_starfish_rbc_round_1_block(&committee, peer)) - .collect::>(); - let parent_references = parents - .iter() - .map(|parent| *parent.reference()) - .collect::>(); - let transactions = vec![BaseTransaction::Share(Transaction::new(vec![7; 64]))]; - let mut encoder = Encoder::new(2, 4, 2).unwrap(); - let encoded = encoder.encode_transactions( - &transactions, - committee.info_length(), - committee.len() - committee.info_length(), - ); - let mut child = VerifiedBlock::new_starfish_rbc( - 1, - 2, - parent_references.clone(), - Vec::new(), - 2, - Vec::new(), - Some(encoded.clone()), - ); - child.preserialize(); - let child = Data::new(child); - let child_reference = *child.reference(); - - let mut transaction_data = TransactionData::new(transactions); - transaction_data.preserialize(); - let (commitment, proof) = - crypto::TransactionsCommitment::new_from_encoded_transactions(&encoded, 0); - let mut shard_data = ProvableShard::new(encoded[0].clone(), 0, proof, commitment); - shard_data.preserialize(); - - // Payload-first arrival is buffered because no dependency-closed - // DagState block exists. It must not be treated as availability. - core.add_transaction_data( - vec![ReconstructedTransactionData { - block_reference: child_reference, - transaction_data, - shard_data, - }], - DataSource::StarfishRbcPayload, - ); - assert_eq!(core.pending_reconstructed_data.len(), 1); - assert!(!rbc_application_is_materialized(&core, child_reference)); - - let (processed, missing, processed_references, _) = - core.add_headers(vec![child], DataSource::BlockBundleStreamingHeader); - assert!(!processed); - assert!(!missing.is_empty()); - assert!(!processed_references.contains(&child_reference)); - assert!(!rbc_application_is_materialized(&core, child_reference)); - - // Adding the parents activates the pending child and atomically - // attaches the buffered payload. HeaderStaged must use this returned - // processed-reference set to emit the delayed availability signal. - let (processed, missing, processed_references, _) = - core.add_headers(parents, DataSource::BlockBundleStreamingHeader); - assert!(processed); - assert!(missing.is_empty()); - assert!(processed_references.contains(&child_reference)); - assert!(core.pending_reconstructed_data.is_empty()); - assert!(rbc_application_is_materialized(&core, child_reference)); - } - #[test] fn mysticeti_bls_non_leader_can_build_round_2_with_prev_leader_parent() { let authority = 0; diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 65405824..b047b191 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -10,11 +10,10 @@ use tokio::sync::{mpsc, oneshot}; use crate::{ block_handler::BlockHandler, bls_certificate_aggregator::CertificateEvent, - core::RbcDagFrontierApplyError, dag_state::DataSource, data::Data, metrics::{Metrics, UtilizationTimerExt}, - starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, + starfish_rbc::PinnedRbcHeader, starfish_rbc_dag_shadow::CommittedFrontierDeltaV1, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ @@ -54,14 +53,6 @@ enum CoreThreadCommand { DataSource, oneshot::Sender<()>, ), - /// Header authority established by the single-owner RBC-DAG carrier - /// actor. This typed command cannot be forged through `BlockBatch.source`. - AddAuthorizedRbcDagHeader( - RbcCanonicalHeader, - oneshot::Sender<(AHashSet, Vec)>, - ), - /// Payload verified against an RBC-DAG-authorized canonical header. - AddAuthorizedRbcDagPayload(ReconstructedTransactionData, oneshot::Sender<()>), MissingParentReferences(oneshot::Sender>), ForceNewBlock(RoundNumber, oneshot::Sender<()>), /// Attempt block creation with relaxed readiness checks (StarfishSpeed soft @@ -83,16 +74,7 @@ enum CoreThreadCommand { ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), /// Commit one deterministic clean carrier-frontier application delta. - ApplyStarfishRbcDagFrontier( - CommittedFrontierDeltaV1, - oneshot::Sender>, - ), - /// Release production only after the ordered recovery bridge processes - /// the service's final startup `Ready` event. - ActivateStarfishRbcDagAuthority(oneshot::Sender<()>), - /// Release the one-outstanding application-production gate after the - /// exact header is durably assigned to a local carrier. - ApplyStarfishRbcDagApplicationAssigned(BlockReference, oneshot::Sender<()>), + ApplyStarfishRbcDagFrontier(CommittedFrontierDeltaV1, oneshot::Sender<()>), /// Store a Sailfish++ timeout certificate in DagState. ApplyTimeoutCert(SailfishTimeoutCert, oneshot::Sender<()>), /// Store a Sailfish++ no-vote certificate in DagState. @@ -117,13 +99,9 @@ impl thread::Result> { + pub fn stop(self) -> Syncer { drop(self.sender); - self.join_handle.join() - } - - pub fn is_finished(&self) -> bool { - self.join_handle.is_finished() + self.join_handle.join().unwrap() } pub async fn add_blocks( @@ -163,23 +141,6 @@ impl (AHashSet, Vec) { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::AddAuthorizedRbcDagHeader(header, sender)) - .await; - receiver.await.expect("core thread is not expected to stop") - } - - pub(crate) async fn add_authorized_rbc_dag_payload(&self, item: ReconstructedTransactionData) { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::AddAuthorizedRbcDagPayload(item, sender)) - .await; - receiver.await.expect("core thread is not expected to stop") - } - pub async fn missing_parent_references(&self) -> Vec { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::MissingParentReferences(sender)) @@ -187,25 +148,6 @@ impl bool { - let (sender, receiver) = oneshot::channel(); - self.metrics.core_lock_enqueued.inc(); - self.metrics.core_queue_length.inc(); - if self - .sender - .send(CoreThreadCommand::MissingParentReferences(sender)) - .await - .is_err() - { - self.metrics.core_queue_length.dec(); - return false; - } - receiver.await.is_ok() - } - pub async fn force_commit(&self) { let (sender, receiver) = oneshot::channel(); self.send(CoreThreadCommand::ForceCommit(sender)).await; @@ -294,25 +236,6 @@ impl CoreThread { self.syncer.add_transaction_data(items, source); sender.send(()).ok(); } - CoreThreadCommand::AddAuthorizedRbcDagHeader(header, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["add_authorized_rbc_dag_header"]) - .inc(); - let result = self.syncer.add_authorized_rbc_dag_header(header); - sender.send(result).ok(); - } - CoreThreadCommand::AddAuthorizedRbcDagPayload(item, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["add_authorized_rbc_dag_payload"]) - .inc(); - self.syncer.add_authorized_rbc_dag_payload(item); - sender.send(()).ok(); - } CoreThreadCommand::MissingParentReferences(sender) => { metrics .core_thread_tasks_total @@ -536,24 +443,7 @@ impl CoreThread { .core_thread_tasks_total .with_label_values(&["apply_starfish_rbc_dag_frontier"]) .inc(); - let result = self.syncer.apply_starfish_rbc_dag_frontier(delta); - sender.send(result).ok(); - } - CoreThreadCommand::ActivateStarfishRbcDagAuthority(sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["activate_starfish_rbc_dag_authority"]) - .inc(); - self.syncer.activate_starfish_rbc_dag_authority(); - sender.send(()).ok(); - } - CoreThreadCommand::ApplyStarfishRbcDagApplicationAssigned(reference, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["apply_starfish_rbc_dag_application_assigned"]) - .inc(); - self.syncer - .apply_starfish_rbc_dag_application_assigned(reference); + self.syncer.apply_starfish_rbc_dag_frontier(delta); sender.send(()).ok(); } CoreThreadCommand::ApplyTimeoutCert(cert, sender) => { @@ -620,7 +510,14 @@ mod tests { Vec::new() } - fn handle_rbc_dag_commit(&mut self, _committed: &[CommittedSubDag]) {} + fn handle_rbc_dag_commit( + &mut self, + _dag_state: &DagState, + _anchor: BlockReference, + _applications: &[BlockReference], + ) -> Vec { + Vec::new() + } fn recover_committed( &mut self, @@ -731,6 +628,6 @@ mod tests { let refs = dispatcher.missing_parent_references().await; assert_eq!(refs, vec![missing_earlier, missing_later]); - assert!(dispatcher.stop().is_ok()); + dispatcher.stop(); } } diff --git a/crates/starfish-core/src/dag_state.rs b/crates/starfish-core/src/dag_state.rs index 0e57fff8..59bbabd6 100644 --- a/crates/starfish-core/src/dag_state.rs +++ b/crates/starfish-core/src/dag_state.rs @@ -84,14 +84,6 @@ pub enum DataSource { RoundGapResponse, /// Transaction data co-carried by the direct Starfish-RBC INIT. StarfishRbcPayload, - /// Header authorized by an authenticated or delivered RBC-DAG carrier. - /// This is an internal provenance label; authorization is carried by the - /// dedicated core-thread command, never by the peer-controlled wire enum. - StarfishRbcDagAuthorizedHeader, - /// Payload verified against an RBC-DAG-authorized header commitment. - /// Like the header label, this is accepted only through the dedicated - /// core-thread command. - StarfishRbcDagAuthorizedPayload, } impl DataSource { @@ -107,8 +99,6 @@ impl DataSource { Self::UnprovableCertificateResponse => "unprovable_certificate_response", Self::RoundGapResponse => "round_gap_response", Self::StarfishRbcPayload => "starfish_rbc_payload", - Self::StarfishRbcDagAuthorizedHeader => "starfish_rbc_dag_authorized_header", - Self::StarfishRbcDagAuthorizedPayload => "starfish_rbc_dag_authorized_payload", } } } @@ -2566,33 +2556,6 @@ impl DagState { self.dag_state_inner.read().last_committed_rounds.clone() } - /// Restore the exact durable RBC-DAG application watermark before the - /// authoritative carrier actor is opened. The receipt is the only commit - /// record for a control-only frontier, so ordinary application-commit - /// recovery cannot reconstruct this vector on its own. - pub(crate) fn restore_rbc_dag_committed_rounds(&self, committed_rounds: &[RoundNumber]) { - let mut inner = self.dag_state_inner.write(); - assert_eq!( - committed_rounds.len(), - inner.committee_size, - "RBC-DAG frontier receipt watermark length must match the committee" - ); - for (authority, (recovered, durable)) in inner - .last_committed_rounds - .iter() - .zip(committed_rounds) - .enumerate() - { - assert!( - durable >= recovered, - "RBC-DAG frontier receipt regresses recovered authority {authority}: durable {durable}, recovered {recovered}" - ); - } - inner - .last_committed_rounds - .clone_from_slice(committed_rounds); - } - pub fn cleanup(&self) { let _timer = self.metrics.dag_state_cleanup_util.utilization_timer(); diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index f118bc4a..aabc76f7 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -7,7 +7,7 @@ use std::{ ops::AddAssign, sync::{ Arc, - atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::Duration, }; @@ -26,9 +26,6 @@ use crate::{ committee::Committee, data::{IN_MEMORY_BLOCKS, IN_MEMORY_BLOCKS_BYTES}, runtime, - starfish_rbc_dag::model::{ - EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, - }, stat::{DivUsize, HistogramSender, PreciseHistogram, histogram}, types::{AuthorityIndex, format_authority_index}, }; @@ -36,43 +33,6 @@ use crate::{ /// Metrics collected by the benchmark. pub const BENCHMARK_DURATION: &str = "benchmark_duration"; -/// One absolute submission window shared by every local-benchmark generator. -/// The common epoch removes sequential-start and polling skew from offered -/// load, cutoff counters, and latency samples. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct BenchmarkTransactionWindow { - pub start: Instant, - pub end: Instant, -} - -impl BenchmarkTransactionWindow { - pub fn new(start: Instant, end: Instant) -> Option { - (start < end).then_some(Self { start, end }) - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[repr(u8)] -pub enum BenchmarkGeneratorState { - Disabled = 0, - Waiting = 1, - Active = 2, - Finished = 3, - Failed = 4, -} - -impl BenchmarkGeneratorState { - pub fn from_u8(value: u8) -> Self { - match value { - 1 => Self::Waiting, - 2 => Self::Active, - 3 => Self::Finished, - 4 => Self::Failed, - _ => Self::Disabled, - } - } -} - pub const TRANSACTION_CERTIFIED_LATENCY: &str = "transaction_certified_latency"; pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; @@ -89,25 +49,7 @@ pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; /// while still detecting an actor that is no longer draining work. pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG: i64 = 4; pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR: i64 = 16; -/// Per-remote-author capacity of authenticated future slots that are inside -/// the executable retention window but outside its immediate admission -/// window. This is an asynchronous safety bound, not a healthy-tail target. -pub const STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_CAPACITY_PER_REMOTE: i64 = - EXECUTABLE_MODEL_BUFFER_WINDOW_V1 as i64 - EXECUTABLE_MODEL_ADMISSION_WINDOW_V1 as i64; -/// With final honest round skew bounded by four and two future rounds admitted, -/// at most two slots per remote author remain buffered in a settled run. -pub const STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_SETTLED_PER_REMOTE: i64 = - STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG - EXECUTABLE_MODEL_ADMISSION_WINDOW_V1 as i64; - -pub const fn starfish_rbc_dag_autonomous_buffered_capacity_bound(committee_size: i64) -> i64 { - STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_CAPACITY_PER_REMOTE - .saturating_mul(committee_size.saturating_sub(1)) -} - -pub const fn starfish_rbc_dag_autonomous_buffered_settled_bound(committee_size: i64) -> i64 { - STARFISH_RBC_DAG_AUTONOMOUS_BUFFERED_SETTLED_PER_REMOTE - .saturating_mul(committee_size.saturating_sub(1)) -} +pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR: i64 = 2; const LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES: &[&str] = &[ "subscribe_broadcast", @@ -133,34 +75,6 @@ const LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES: &[&str] = &[ "rbc_dag_shadow_carrier_response", "rbc_dag_shadow_carrier_sync_request", "rbc_dag_shadow_carrier_sync_response", - "rbc_dag_application_payload_request", - "rbc_dag_application_payload_response", -]; - -pub(crate) const RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT: &str = "creation_to_assignment"; -pub(crate) const RBC_DAG_LATENCY_CREATION_TO_DELIVERY: &str = "creation_to_delivery"; -pub(crate) const RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED: &str = - "creation_to_frontier_generated"; -pub(crate) const RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED: &str = - "creation_to_frontier_applied"; - -const RBC_DAG_PIPELINE_LATENCY_STAGES: &[&str] = &[ - RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, - RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, - RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, -]; -pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD: &str = "physical_forward"; -pub(crate) const RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD: &str = "physical_backward"; -const RBC_DAG_COMMIT_DISTANCE_KINDS: &[&str] = &[ - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, -]; -const RBC_DAG_PROJECTION_HOL_STATES: &[&str] = &[ - "insufficient_lookahead", - "direct_evidence_pending", - "awaiting_indirect_anchor", - "ready", ]; #[derive(Clone)] @@ -170,10 +84,6 @@ pub struct Metrics { pub leader_timeout_total: IntCounter, pub proposal_wait_time_total_us: IntCounter, pub sequenced_transactions_total: IntCounter, - /// Transactions committed before the coordinated benchmark's exact - /// monotonic cutoff. The ordinary sequenced counter remains open through - /// the bounded drain to measure eventual active-window throughput. - pub sequenced_transactions_cutoff_total: IntCounter, pub sequenced_transactions_bytes: IntCounter, pub sailfish_rbc_fast_total: IntCounter, pub sailfish_rbc_slow_total: IntCounter, @@ -290,40 +200,6 @@ pub struct Metrics { pub starfish_rbc_dag_shadow_buffered_authenticated: IntGauge, pub starfish_rbc_dag_projected_vertices_total: IntCounter, pub starfish_rbc_dag_projection_decisions_total: IntCounterVec, - /// Active-window application latency decomposed by one of four fixed - /// pipeline stages. Keeping sum/count/max avoids per-block labels and the - /// cost of a high-volume histogram on the carrier actor's hot path. - pub starfish_rbc_dag_pipeline_latency_ns_total: IntCounterVec, - pub starfish_rbc_dag_pipeline_latency_samples_total: IntCounterVec, - pub starfish_rbc_dag_pipeline_latency_ns_max: IntGaugeVec, - /// Diagnostic-only round distances for first-committed applications. - /// Physical deltas use separate forward/backward magnitude labels so - /// cross-author clock skew is not hidden by unsigned saturation. - pub starfish_rbc_dag_commit_distance_rounds_total: IntCounterVec, - pub starfish_rbc_dag_commit_distance_samples_total: IntCounterVec, - pub starfish_rbc_dag_commit_distance_rounds_max: IntGaugeVec, - /// Current and process-high-water queue depths, with the bounded labels - /// `local` and `projection`. - pub starfish_rbc_dag_pipeline_queue_depth: IntGaugeVec, - pub starfish_rbc_dag_pipeline_queue_depth_max: IntGaugeVec, - pub starfish_rbc_dag_highest_projected_consensus_round: IntGauge, - pub starfish_rbc_dag_next_undecided_consensus_round: IntGauge, - pub starfish_rbc_dag_next_undecided_projected_stake: IntGauge, - pub starfish_rbc_dag_last_committed_consensus_round: IntGauge, - /// One-hot current projection head-of-line state. The label vocabulary is - /// fixed in `set_starfish_rbc_dag_pipeline_state`. - pub starfish_rbc_dag_projection_hol_state: IntGaugeVec, - /// Frontier lifecycle counters plus the current/high-water number created - /// by the synchronous carrier actor but not yet applied by the core - /// dispatcher. - pub starfish_rbc_dag_frontier_events_total: IntCounterVec, - pub starfish_rbc_dag_frontiers_inflight: IntGauge, - pub starfish_rbc_dag_frontiers_inflight_max: IntGauge, - /// Sequenced-transaction total observed by the frontier bridge only after - /// it records the corresponding final application-latency samples. The - /// local benchmark uses this release/acquire acknowledgement before - /// closing its post-cutoff transaction-observation gate. - starfish_rbc_dag_frontier_applied_sequenced_transactions: Arc, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -359,22 +235,11 @@ pub struct Metrics { /// True iff the validator is inside the active transaction-submission /// window. Outside this window — during the warmup before the first - /// transaction is generated, and after the generator stops — protocol - /// throughput/latency metrics and the `benchmark_duration` clock are - /// skipped. Transaction commits use `transaction_metrics_active` so the - /// offered window can be followed through a bounded drain. + /// transaction is generated, and after the generator stops — every + /// rate-relevant metric update (latency observations, sequenced / + /// committed counters, the `benchmark_duration` clock) is skipped, so + /// reported TPS / BPS / p50 latency reflect only the steady-state window. pub metrics_active: Arc, - /// Transaction and application-pipeline observations remain enabled - /// during the bounded post-window drain, while all ordinary - /// protocol/window metrics close exactly at the shared cutoff. This makes - /// active-window latency uncensored without charging drain traffic to - /// block/RBC throughput. - pub transaction_metrics_active: Arc, - /// Runtime-only coordinated generator lifecycle for local benchmarks. - pub benchmark_generator_state: Arc, - /// Common benchmark cutoff expressed in microseconds since this - /// validator's `validator_start`; zero outside coordinated benchmarks. - pub benchmark_transaction_cutoff_micros: Arc, /// Wall-clock instant the validator's metrics were first activated, in /// microseconds since `validator_start`. Used by the /// `benchmark_duration` Prometheus counter so its denominator counts @@ -414,38 +279,6 @@ pub struct AutonomousClockBenchmarkBaseline { carrier_round: i64, } -/// Immutable per-validator RBC-DAG state sampled at the common transaction -/// cutoff. The bounded post-window transaction drain may improve or worsen -/// live gauges, but it must never rewrite the verdict for the measured -/// interval. -#[derive(Clone, Copy, Debug, Default)] -pub struct AutonomousClockBenchmarkSnapshot { - accepted_heartbeats: u64, - accepted_application_carriers: u64, - delivered_carriers: u64, - delivered_applications: u64, - committed_frontiers: u64, - frontier_applications: u64, - projected_vertices: u64, - projection_decisions: u64, - wal_batches: u64, - wal_records: u64, - clock_valid: i64, - carrier_round: i64, - phase_backlog: i64, - admitted_authors: i64, - admitted_stake: i64, - buffered_authenticated: i64, - pending_recovery: i64, -} - -impl AutonomousClockBenchmarkSnapshot { - fn accepted_local_carriers(self) -> u64 { - self.accepted_heartbeats - .saturating_add(self.accepted_application_carriers) - } -} - /// Per-validator cumulative counters sampled at the exact start of a local /// benchmark's active transaction window. Rates subtract this snapshot so /// connection warmup and shadow-WAL replay are not charged to the protocol. @@ -458,20 +291,6 @@ pub struct LocalBenchmarkCounterBaseline { outbound_messages: Vec<(u64, u64)>, } -#[derive(Clone, Copy, Debug)] -pub struct LocalBenchmarkTransactionOutcome { - /// Exact successful sends across all honest local generators in the - /// shared active window. Byzantine generators are disabled by the local - /// harness, so this is the global set every honest validator must drain. - pub offered_transactions: u64, - /// Mean per-honest-validator commits observed at the common cutoff. - pub cutoff_committed_transactions: u64, - /// Mean per-honest-validator commits after the bounded drain. - pub eventual_committed_transactions: u64, - pub drain_elapsed: Duration, - pub drain_complete: bool, -} - #[derive(Debug, Eq, PartialEq)] struct AutonomousClockBenchmarkSummary { valid_nodes: usize, @@ -502,134 +321,200 @@ fn summarize_autonomous_clock_benchmark( metrics: &[Arc], committee_size: usize, baselines: Option<&[AutonomousClockBenchmarkBaseline]>, - cutoff_snapshots: Option<&[AutonomousClockBenchmarkSnapshot]>, embedded_rbc_authority: bool, ) -> AutonomousClockBenchmarkSummary { let committee_size = i64::try_from(committee_size).unwrap_or(i64::MAX); let maximum_phase_backlog_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR.saturating_mul(committee_size); let maximum_buffered_authenticated_bound = - starfish_rbc_dag_autonomous_buffered_settled_bound(committee_size); - - // A supplied cutoff vector is authoritative. Missing entries fail closed - // to the all-zero default rather than falling back to mutable drain-time - // gauges and accidentally turning an invalid measured run into VALID. - let observations = metrics - .iter() - .enumerate() - .map(|(index, metrics)| match cutoff_snapshots { - Some(snapshots) => snapshots.get(index).copied().unwrap_or_default(), - None => metrics.autonomous_clock_benchmark_snapshot(), - }) - .collect::>(); + STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR.saturating_mul(committee_size); - let valid_nodes = observations + let valid_nodes = metrics .iter() - .filter(|snapshot| snapshot.clock_valid == 1) + .filter(|metrics| metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1) .count(); - let progress_nodes = observations + let progress_nodes = metrics .iter() .enumerate() - .filter(|(index, snapshot)| { + .filter(|(index, metrics)| { let baseline = baselines .and_then(|baselines| baselines.get(*index)) .copied() .unwrap_or_default(); - snapshot.accepted_local_carriers() > baseline.accepted_local_carriers - && snapshot.delivered_carriers > baseline.delivered_carriers + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .get() + .saturating_add( + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["application_carrier", "accepted"]) + .get(), + ) + > baseline.accepted_local_carriers + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get() + > baseline.delivered_carriers && (!embedded_rbc_authority - || snapshot.delivered_applications > baseline.delivered_applications - && snapshot.committed_frontiers > baseline.committed_frontiers - && snapshot.frontier_applications > baseline.frontier_applications) - && snapshot.projected_vertices > baseline.projected_vertices - && snapshot.projection_decisions > baseline.projection_decisions - && snapshot.wal_batches > baseline.wal_batches - && snapshot.wal_records > baseline.wal_records - && snapshot.carrier_round > baseline.carrier_round + || metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .get() + > baseline.delivered_applications + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get() + > baseline.committed_frontiers + && metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get() + > baseline.frontier_applications) + && metrics.starfish_rbc_dag_projected_vertices_total.get() + > baseline.projected_vertices + && metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get() + > baseline.projection_decisions + && metrics + .starfish_rbc_dag_shadow_wal_appended_batches_total + .get() + > baseline.wal_batches + && metrics + .starfish_rbc_dag_shadow_wal_appended_records_total + .get() + > baseline.wal_records + && metrics.starfish_rbc_dag_shadow_carrier_round.get() > baseline.carrier_round }) .count(); - let bounded_nodes = observations + let bounded_nodes = metrics .iter() - .filter(|snapshot| { - snapshot.phase_backlog >= 0 - && snapshot.phase_backlog <= maximum_phase_backlog_bound - && snapshot.admitted_authors >= 0 - && snapshot.admitted_authors <= committee_size - && snapshot.admitted_stake >= 0 - && snapshot.buffered_authenticated >= 0 - && snapshot.buffered_authenticated <= maximum_buffered_authenticated_bound - && snapshot.pending_recovery == 0 + .filter(|metrics| { + let phase_backlog = metrics.starfish_rbc_dag_shadow_phase_backlog.get(); + let admitted_authors = metrics.starfish_rbc_dag_shadow_admitted_authors.get(); + let admitted_stake = metrics.starfish_rbc_dag_shadow_admitted_stake.get(); + let buffered = metrics.starfish_rbc_dag_shadow_buffered_authenticated.get(); + phase_backlog >= 0 + && phase_backlog <= maximum_phase_backlog_bound + && admitted_authors >= 0 + && admitted_authors <= committee_size + && admitted_stake >= 0 + && buffered >= 0 + && buffered <= maximum_buffered_authenticated_bound + && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 }) .count(); - let accepted_heartbeats = observations + let accepted_heartbeats = metrics .iter() - .map(|snapshot| snapshot.accepted_heartbeats) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["heartbeat", "accepted"]) + .get() + }) .sum(); - let delivered_carriers = observations + let delivered_carriers = metrics .iter() - .map(|snapshot| snapshot.delivered_carriers) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "shadow"]) + .get() + }) .sum(); - let delivered_applications = observations + let delivered_applications = metrics .iter() - .map(|snapshot| snapshot.delivered_applications) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["delivery", "embedded_application"]) + .get() + }) .sum(); - let committed_frontiers = observations + let committed_frontiers = metrics .iter() - .map(|snapshot| snapshot.committed_frontiers) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .get() + }) .sum(); - let frontier_applications = observations + let frontier_applications = metrics .iter() - .map(|snapshot| snapshot.frontier_applications) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .get() + }) .sum(); - let projected_vertices = observations + let projected_vertices = metrics .iter() - .map(|snapshot| snapshot.projected_vertices) + .map(|metrics| metrics.starfish_rbc_dag_projected_vertices_total.get()) .sum(); - let projection_decisions = observations + let projection_decisions = metrics .iter() - .map(|snapshot| snapshot.projection_decisions) + .map(|metrics| { + metrics + .starfish_rbc_dag_projection_decisions_total + .with_label_values(&["direct_commit"]) + .get() + }) .sum(); - let wal_batches = observations + let wal_batches = metrics .iter() - .map(|snapshot| snapshot.wal_batches) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_wal_appended_batches_total + .get() + }) .sum(); - let wal_records = observations + let wal_records = metrics .iter() - .map(|snapshot| snapshot.wal_records) + .map(|metrics| { + metrics + .starfish_rbc_dag_shadow_wal_appended_records_total + .get() + }) .sum(); - let pending_recovery = observations + let pending_recovery = metrics .iter() - .map(|snapshot| snapshot.pending_recovery) + .map(|metrics| metrics.starfish_rbc_dag_shadow_pending_recovery.get()) .sum(); - let minimum_round = observations + let minimum_round = metrics .iter() - .map(|snapshot| snapshot.carrier_round) + .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) .min() .unwrap_or_default(); - let maximum_round = observations + let maximum_round = metrics .iter() - .map(|snapshot| snapshot.carrier_round) + .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) .max() .unwrap_or_default(); - let maximum_phase_backlog = observations + let maximum_phase_backlog = metrics .iter() - .map(|snapshot| snapshot.phase_backlog) + .map(|metrics| metrics.starfish_rbc_dag_shadow_phase_backlog.get()) .max() .unwrap_or_default(); - let maximum_admitted_authors = observations + let maximum_admitted_authors = metrics .iter() - .map(|snapshot| snapshot.admitted_authors) + .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_authors.get()) .max() .unwrap_or_default(); - let maximum_admitted_stake = observations + let maximum_admitted_stake = metrics .iter() - .map(|snapshot| snapshot.admitted_stake) + .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_stake.get()) .max() .unwrap_or_default(); - let maximum_buffered_authenticated = observations + let maximum_buffered_authenticated = metrics .iter() - .map(|snapshot| snapshot.buffered_authenticated) + .map(|metrics| metrics.starfish_rbc_dag_shadow_buffered_authenticated.get()) .max() .unwrap_or_default(); let round_lag = maximum_round.saturating_sub(minimum_round); @@ -678,176 +563,18 @@ pub struct VecHistogramReporter { gauge: IntGaugeVec, } -fn set_gauge_max(gauge: &IntGauge, value: i64) { - if value > gauge.get() { - gauge.set(value); - } -} - -fn format_rbc_dag_round_distance(total: u64, samples: u64, maximum: i64) -> String { - let average = if samples == 0 { - 0.0 - } else { - total as f64 / samples as f64 - }; - format!("{average:.2}/{maximum} rounds (n={samples})") -} - impl Metrics { - pub(crate) fn observe_starfish_rbc_dag_pipeline_latency_ns( - &self, - stage: &'static str, - total_ns: u64, - samples: u64, - max_ns: u64, - ) { - debug_assert!(RBC_DAG_PIPELINE_LATENCY_STAGES.contains(&stage)); - // These stages follow the finite set of applications offered during - // the common transaction window. Keep them open through the bounded - // drain so delivery/frontier latency is not right-censored at the - // submission cutoff. Protocol rates and round-distance observations - // remain scoped by `metrics_active` below. - if samples == 0 || !self.transaction_metrics_active.load(Ordering::Relaxed) { - return; - } - self.starfish_rbc_dag_pipeline_latency_ns_total - .with_label_values(&[stage]) - .inc_by(total_ns); - self.starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[stage]) - .inc_by(samples); - set_gauge_max( - &self - .starfish_rbc_dag_pipeline_latency_ns_max - .with_label_values(&[stage]), - i64::try_from(max_ns).unwrap_or(i64::MAX), - ); - } - - pub(crate) fn observe_starfish_rbc_dag_commit_round_distance( - &self, - kind: &'static str, - total_rounds: u64, - samples: u64, - max_rounds: u64, - ) { - debug_assert!(RBC_DAG_COMMIT_DISTANCE_KINDS.contains(&kind)); - if samples == 0 || !self.metrics_active.load(Ordering::Relaxed) { - return; - } - self.starfish_rbc_dag_commit_distance_rounds_total - .with_label_values(&[kind]) - .inc_by(total_rounds); - self.starfish_rbc_dag_commit_distance_samples_total - .with_label_values(&[kind]) - .inc_by(samples); - set_gauge_max( - &self - .starfish_rbc_dag_commit_distance_rounds_max - .with_label_values(&[kind]), - i64::try_from(max_rounds).unwrap_or(i64::MAX), - ); - } - - pub(crate) fn set_starfish_rbc_dag_pipeline_state( - &self, - pending_local: usize, - pending_projection: usize, - highest_projected_round: u32, - next_undecided_round: u32, - next_undecided_projected_stake: u64, - last_committed_round: u32, - hol_state: &'static str, - ) { - debug_assert!(RBC_DAG_PROJECTION_HOL_STATES.contains(&hol_state)); - for (queue, depth) in [("local", pending_local), ("projection", pending_projection)] { - let depth = i64::try_from(depth).unwrap_or(i64::MAX); - self.starfish_rbc_dag_pipeline_queue_depth - .with_label_values(&[queue]) - .set(depth); - set_gauge_max( - &self - .starfish_rbc_dag_pipeline_queue_depth_max - .with_label_values(&[queue]), - depth, - ); - } - self.starfish_rbc_dag_highest_projected_consensus_round - .set(i64::from(highest_projected_round)); - self.starfish_rbc_dag_next_undecided_consensus_round - .set(i64::from(next_undecided_round)); - self.starfish_rbc_dag_next_undecided_projected_stake - .set(i64::try_from(next_undecided_projected_stake).unwrap_or(i64::MAX)); - self.starfish_rbc_dag_last_committed_consensus_round - .set(i64::from(last_committed_round)); - for state in RBC_DAG_PROJECTION_HOL_STATES { - self.starfish_rbc_dag_projection_hol_state - .with_label_values(&[state]) - .set(i64::from(*state == hol_state)); - } - } - - pub(crate) fn starfish_rbc_dag_frontier_generated(&self) { - self.starfish_rbc_dag_frontier_events_total - .with_label_values(&["generated"]) - .inc(); - self.starfish_rbc_dag_frontiers_inflight.inc(); - set_gauge_max( - &self.starfish_rbc_dag_frontiers_inflight_max, - self.starfish_rbc_dag_frontiers_inflight.get(), - ); - } - - pub(crate) fn starfish_rbc_dag_frontier_applied(&self) { - self.starfish_rbc_dag_frontier_events_total - .with_label_values(&["applied"]) - .inc(); - if self.starfish_rbc_dag_frontiers_inflight.get() > 0 { - self.starfish_rbc_dag_frontiers_inflight.dec(); - } - // `RbcDagAppliedFrontierObservationV1::observe` calls this only after - // recording creation-to-frontier-applied latency. Publishing the - // sequenced count last gives the benchmark an ordered drain barrier, - // without confusing application samples (blocks) with transactions. - self.starfish_rbc_dag_frontier_applied_sequenced_transactions - .store(self.sequenced_transactions_total.get(), Ordering::Release); - } - - pub(crate) fn starfish_rbc_dag_frontier_ignored(&self) { - self.starfish_rbc_dag_frontier_events_total - .with_label_values(&["ignored"]) - .inc(); - if self.starfish_rbc_dag_frontiers_inflight.get() > 0 { - self.starfish_rbc_dag_frontiers_inflight.dec(); - } - } - pub fn autonomous_clock_benchmark_baseline(&self) -> AutonomousClockBenchmarkBaseline { - let snapshot = self.autonomous_clock_benchmark_snapshot(); AutonomousClockBenchmarkBaseline { - accepted_local_carriers: snapshot.accepted_local_carriers(), - delivered_carriers: snapshot.delivered_carriers, - delivered_applications: snapshot.delivered_applications, - committed_frontiers: snapshot.committed_frontiers, - frontier_applications: snapshot.frontier_applications, - projected_vertices: snapshot.projected_vertices, - projection_decisions: snapshot.projection_decisions, - wal_batches: snapshot.wal_batches, - wal_records: snapshot.wal_records, - carrier_round: snapshot.carrier_round, - } - } - - pub fn autonomous_clock_benchmark_snapshot(&self) -> AutonomousClockBenchmarkSnapshot { - AutonomousClockBenchmarkSnapshot { - accepted_heartbeats: self + accepted_local_carriers: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["heartbeat", "accepted"]) - .get(), - accepted_application_carriers: self - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["application_carrier", "accepted"]) - .get(), + .get() + .saturating_add( + self.starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["application_carrier", "accepted"]) + .get(), + ), delivered_carriers: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) @@ -875,30 +602,10 @@ impl Metrics { wal_records: self .starfish_rbc_dag_shadow_wal_appended_records_total .get(), - clock_valid: self.starfish_rbc_dag_shadow_clock_valid.get(), carrier_round: self.starfish_rbc_dag_shadow_carrier_round.get(), - phase_backlog: self.starfish_rbc_dag_shadow_phase_backlog.get(), - admitted_authors: self.starfish_rbc_dag_shadow_admitted_authors.get(), - admitted_stake: self.starfish_rbc_dag_shadow_admitted_stake.get(), - buffered_authenticated: self.starfish_rbc_dag_shadow_buffered_authenticated.get(), - pending_recovery: self.starfish_rbc_dag_shadow_pending_recovery.get(), } } - /// Number of offered applications whose authoritative RBC-DAG frontier - /// has been applied locally and whose final pipeline-latency sample has - /// therefore already been recorded. - pub fn starfish_rbc_dag_frontier_applied_latency_samples(&self) -> u64 { - self.starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED]) - .get() - } - - pub fn starfish_rbc_dag_frontier_applied_sequenced_transactions(&self) -> u64 { - self.starfish_rbc_dag_frontier_applied_sequenced_transactions - .load(Ordering::Acquire) - } - pub fn local_benchmark_counter_baseline(&self) -> LocalBenchmarkCounterBaseline { LocalBenchmarkCounterBaseline { sequenced_transactions: self.sequenced_transactions_total.get(), @@ -1363,122 +1070,6 @@ impl Metrics { registry, ) .unwrap(), - starfish_rbc_dag_pipeline_latency_ns_total: - register_int_counter_vec_with_registry!( - "starfish_rbc_dag_pipeline_latency_ns_total", - "Active-window application latency nanoseconds accumulated by bounded RBC-DAG pipeline stage", - &["stage"], - registry, - ) - .unwrap(), - starfish_rbc_dag_pipeline_latency_samples_total: - register_int_counter_vec_with_registry!( - "starfish_rbc_dag_pipeline_latency_samples_total", - "Active-window application latency sample count by bounded RBC-DAG pipeline stage", - &["stage"], - registry, - ) - .unwrap(), - starfish_rbc_dag_pipeline_latency_ns_max: register_int_gauge_vec_with_registry!( - "starfish_rbc_dag_pipeline_latency_ns_max", - "Maximum active-window application latency nanoseconds by bounded RBC-DAG pipeline stage", - &["stage"], - registry, - ) - .unwrap(), - starfish_rbc_dag_commit_distance_rounds_total: - register_int_counter_vec_with_registry!( - "starfish_rbc_dag_commit_distance_rounds_total", - "Active-window round-distance magnitude accumulated for first-committed RBC-DAG applications", - &["kind"], - registry, - ) - .unwrap(), - starfish_rbc_dag_commit_distance_samples_total: - register_int_counter_vec_with_registry!( - "starfish_rbc_dag_commit_distance_samples_total", - "Active-window first-committed RBC-DAG application sample count by round-distance kind", - &["kind"], - registry, - ) - .unwrap(), - starfish_rbc_dag_commit_distance_rounds_max: - register_int_gauge_vec_with_registry!( - "starfish_rbc_dag_commit_distance_rounds_max", - "Maximum active-window first-committed RBC-DAG application round-distance magnitude", - &["kind"], - registry, - ) - .unwrap(), - starfish_rbc_dag_pipeline_queue_depth: register_int_gauge_vec_with_registry!( - "starfish_rbc_dag_pipeline_queue_depth", - "Current RBC-DAG pipeline queue depth by bounded queue name", - &["queue"], - registry, - ) - .unwrap(), - starfish_rbc_dag_pipeline_queue_depth_max: register_int_gauge_vec_with_registry!( - "starfish_rbc_dag_pipeline_queue_depth_max", - "Process-high-water RBC-DAG pipeline queue depth by bounded queue name", - &["queue"], - registry, - ) - .unwrap(), - starfish_rbc_dag_highest_projected_consensus_round: - register_int_gauge_with_registry!( - "starfish_rbc_dag_highest_projected_consensus_round", - "Highest clean consensus round projected by this RBC-DAG runtime", - registry, - ) - .unwrap(), - starfish_rbc_dag_next_undecided_consensus_round: - register_int_gauge_with_registry!( - "starfish_rbc_dag_next_undecided_consensus_round", - "Oldest clean projected consensus round not yet decided", - registry, - ) - .unwrap(), - starfish_rbc_dag_next_undecided_projected_stake: - register_int_gauge_with_registry!( - "starfish_rbc_dag_next_undecided_projected_stake", - "Projected distinct-author stake at the oldest undecided consensus round", - registry, - ) - .unwrap(), - starfish_rbc_dag_last_committed_consensus_round: - register_int_gauge_with_registry!( - "starfish_rbc_dag_last_committed_consensus_round", - "Highest consensus round whose committed frontier was generated", - registry, - ) - .unwrap(), - starfish_rbc_dag_projection_hol_state: register_int_gauge_vec_with_registry!( - "starfish_rbc_dag_projection_hol_state", - "One-hot current certified-projection head-of-line state", - &["reason"], - registry, - ) - .unwrap(), - starfish_rbc_dag_frontier_events_total: register_int_counter_vec_with_registry!( - "starfish_rbc_dag_frontier_events_total", - "Committed RBC-DAG frontier lifecycle events by bounded stage", - &["stage"], - registry, - ) - .unwrap(), - starfish_rbc_dag_frontiers_inflight: register_int_gauge_with_registry!( - "starfish_rbc_dag_frontiers_inflight", - "Committed RBC-DAG frontiers generated but not yet applied or ignored", - registry, - ) - .unwrap(), - starfish_rbc_dag_frontiers_inflight_max: register_int_gauge_with_registry!( - "starfish_rbc_dag_frontiers_inflight_max", - "Process-high-water committed RBC-DAG frontiers awaiting event application", - registry, - ) - .unwrap(), - starfish_rbc_dag_frontier_applied_sequenced_transactions: Arc::new(AtomicU64::new(0)), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1578,12 +1169,6 @@ impl Metrics { registry, ) .unwrap(), - sequenced_transactions_cutoff_total: register_int_counter_with_registry!( - "sequenced_transactions_cutoff_total", - "Transactions sequenced before the coordinated benchmark cutoff", - registry, - ) - .unwrap(), sequenced_transactions_bytes: register_int_counter_with_registry!( "sequenced_transactions_bytes", "Total bytes of sequenced transactions", @@ -1930,11 +1515,6 @@ impl Metrics { // bound). The transaction generator overrides to false during // its warmup when the orchestrator sets a finite duration. metrics_active: Arc::new(AtomicBool::new(true)), - transaction_metrics_active: Arc::new(AtomicBool::new(true)), - benchmark_generator_state: Arc::new(AtomicU8::new( - BenchmarkGeneratorState::Disabled as u8, - )), - benchmark_transaction_cutoff_micros: Arc::new(AtomicU64::new(0)), active_start_micros: Arc::new(AtomicU64::new(0)), validator_start: tokio::time::Instant::now(), }; @@ -1951,10 +1531,7 @@ impl Metrics { starfish_rbc_dag_autonomous_clock_expected: bool, starfish_rbc_dag_embedded_rbc_authority_expected: bool, autonomous_clock_baselines: Option>, - autonomous_clock_cutoffs: Option>, counter_baselines: Option>, - counter_cutoffs: Option>, - transaction_outcome: Option, ) { let num_validators = metrics.len() as u64; @@ -1973,27 +1550,19 @@ impl Metrics { }) .sum::() / num_validators; - let cutoff_transactions = transaction_outcome - .map(|outcome| outcome.cutoff_committed_transactions) - .unwrap_or(average_transactions); - let average_tps = cutoff_transactions as f64 / duration_secs as f64; + let average_tps = average_transactions as f64 / duration_secs as f64; let average_blocks_submitted = metrics .iter() .enumerate() .map(|(index, metrics)| { - counter_cutoffs - .as_ref() - .and_then(|cutoffs| cutoffs.get(index)) - .map(|cutoff| cutoff.dag_state_entries) - .unwrap_or_else(|| metrics.dag_state_entries.get()) - .saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.dag_state_entries) - .unwrap_or_default(), - ) + metrics.dag_state_entries.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.dag_state_entries) + .unwrap_or_default(), + ) }) .sum::() / num_validators; @@ -2003,18 +1572,13 @@ impl Metrics { .iter() .enumerate() .map(|(index, metrics)| { - counter_cutoffs - .as_ref() - .and_then(|cutoffs| cutoffs.get(index)) - .map(|cutoff| cutoff.bytes_sent) - .unwrap_or_else(|| metrics.bytes_sent_total.get()) - .saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.bytes_sent) - .unwrap_or_default(), - ) + metrics.bytes_sent_total.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.bytes_sent) + .unwrap_or_default(), + ) }) .sum::() / num_validators; @@ -2022,18 +1586,13 @@ impl Metrics { .iter() .enumerate() .map(|(index, metrics)| { - counter_cutoffs - .as_ref() - .and_then(|cutoffs| cutoffs.get(index)) - .map(|cutoff| cutoff.bytes_received) - .unwrap_or_else(|| metrics.bytes_received_total.get()) - .saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.bytes_received) - .unwrap_or_default(), - ) + metrics.bytes_received_total.get().saturating_sub( + counter_baselines + .as_ref() + .and_then(|baselines| baselines.get(index)) + .map(|baseline| baseline.bytes_received) + .unwrap_or_default(), + ) }) .sum::() / num_validators; @@ -2120,43 +1679,13 @@ impl Metrics { table.add_row(row![bH2->""]); table.add_row(row![bH2->"Performance Metrics"]); table.add_row( - row![b->"p50 block latency:", format!("{:.2} millis", p50_block_committed_latency)], + row![b->"Average block latency:", format!("{:.2} millis", p50_block_committed_latency)], ); table.add_row(row![ - b->"p50 e2e latency:", + b->"Average e2e latency:", format!("{:.2} millis", p50_transaction_committed_latency) ]); - if let Some(outcome) = transaction_outcome { - table.add_row(row![ - b->"Offered TPS:", - format!( - "{:.2} tx/s ({} exact successful submissions)", - outcome.offered_transactions as f64 / duration_secs as f64, - outcome.offered_transactions, - ) - ]); - table.add_row(row![ - b->"Committed TPS at cutoff:", - format!("{:.2} tx/s", outcome.cutoff_committed_transactions as f64 / duration_secs as f64) - ]); - table.add_row(row![ - b->"Eventual active-window TPS:", - format!( - "{:.2} tx/s ({}, drain {:.2}s)", - outcome.eventual_committed_transactions as f64 / duration_secs as f64, - if outcome.drain_complete { "complete" } else { "INCOMPLETE" }, - outcome.drain_elapsed.as_secs_f64(), - ) - ]); - table.add_row(row![ - b->"Cutoff backlog:", - outcome - .offered_transactions - .saturating_sub(outcome.cutoff_committed_transactions) - ]); - } else { - table.add_row(row![b->"Average TPS:", format!("{:.2} tx/s", average_tps)]); - } + table.add_row(row![b->"Average TPS:", format!("{:.2} tx/s", average_tps)]); table.add_row(row![b->"Average BPS:", format!("{:.2} blocks/s", average_bps)]); // Network metrics @@ -2180,17 +1709,10 @@ impl Metrics { .iter() .enumerate() .map(|(validator_index, metrics)| { - let current = counter_cutoffs - .as_ref() - .and_then(|cutoffs| cutoffs.get(validator_index)) - .and_then(|cutoff| cutoff.outbound_messages.get(message_index)) - .map(|(bytes, _)| *bytes) - .unwrap_or_else(|| { - metrics - .network_message_bytes_sent_total - .with_label_values(&[request_type]) - .get() - }); + let current = metrics + .network_message_bytes_sent_total + .with_label_values(&[request_type]) + .get(); let baseline = counter_baselines .as_ref() .and_then(|baselines| baselines.get(validator_index)) @@ -2208,17 +1730,10 @@ impl Metrics { .iter() .enumerate() .map(|(validator_index, metrics)| { - let current = counter_cutoffs - .as_ref() - .and_then(|cutoffs| cutoffs.get(validator_index)) - .and_then(|cutoff| cutoff.outbound_messages.get(message_index)) - .map(|(_, requests)| *requests) - .unwrap_or_else(|| { - metrics - .network_requests_sent_total - .with_label_values(&[request_type]) - .get() - }); + let current = metrics + .network_requests_sent_total + .with_label_values(&[request_type]) + .get(); let baseline = counter_baselines .as_ref() .and_then(|baselines| baselines.get(validator_index)) @@ -2250,14 +1765,8 @@ impl Metrics { } } let total_average_transactions = (average_tps * duration_secs as f64) as u64; - // Every honest validator sequences the same global offered set, so - // its bandwidth denominator is the aggregate submissions across all - // generators—not one generator's local share. - let offered_global = transaction_outcome - .map(|outcome| outcome.offered_transactions as f64) - .unwrap_or(total_average_transactions as f64); - let bandwidth_efficiency = if offered_global > 0.0 { - average_bytes_sent as f64 / offered_global / 512.0 + let bandwidth_efficiency = if total_average_transactions > 0 { + average_bytes_sent as f64 / total_average_transactions as f64 / 512.0 } else { 0.0 }; @@ -2268,7 +1777,6 @@ impl Metrics { &metrics, committee_size, autonomous_clock_baselines.as_deref(), - autonomous_clock_cutoffs.as_deref(), starfish_rbc_dag_embedded_rbc_authority_expected, ); let round_lag = summary.maximum_round.saturating_sub(summary.minimum_round); @@ -2282,7 +1790,7 @@ impl Metrics { } ]); table.add_row(row![ - b->"Cutoff clock verdict:", + b->"Clock verdict:", if summary.verdict_valid { "VALID".to_owned() } else { @@ -2290,7 +1798,7 @@ impl Metrics { } ]); table.add_row(row![ - b->"Cutoff valid/progress/bounded validators:", + b->"Valid/progress/bounded validators:", format!( "{}/{}, {}/{}, {}/{}", summary.valid_nodes, @@ -2302,7 +1810,7 @@ impl Metrics { ) ]); table.add_row(row![ - b->"Cutoff clock/WAL progress:", + b->"Clock/WAL progress:", format!( "heartbeats={}, carrier deliveries={}, application deliveries={}, committed frontiers={}, frontier applications={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, @@ -2319,7 +1827,7 @@ impl Metrics { ) ]); table.add_row(row![ - b->"Cutoff bounded state:", + b->"Bounded live state:", format!( "round skew={round_lag}/{}, max phase backlog={}/{}, admitted authors={}/{}, stake={}, max buffered={}/{}, pending recovery={}", STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, @@ -2333,328 +1841,6 @@ impl Metrics { summary.pending_recovery, ) ]); - let cutoff_per_validator_progress = metrics - .iter() - .enumerate() - .map(|(index, metrics)| { - let baseline = autonomous_clock_baselines - .as_deref() - .and_then(|baselines| baselines.get(index)) - .copied() - .unwrap_or_default(); - let snapshot = autonomous_clock_cutoffs - .as_deref() - .and_then(|snapshots| snapshots.get(index)) - .copied() - .unwrap_or_else(|| metrics.autonomous_clock_benchmark_snapshot()); - let applications = snapshot - .delivered_applications - .saturating_sub(baseline.delivered_applications); - let frontiers = snapshot - .committed_frontiers - .saturating_sub(baseline.committed_frontiers); - format!( - "{index}:r{}/a{applications}/f{frontiers}/v{}", - snapshot.carrier_round, snapshot.clock_valid, - ) - }) - .collect::>() - .join(", "); - table.add_row(row![ - b->"Cutoff per-validator round/app/frontier/valid:", - cutoff_per_validator_progress, - ]); - let final_per_validator_state = metrics - .iter() - .enumerate() - .map(|(index, metrics)| { - let snapshot = metrics.autonomous_clock_benchmark_snapshot(); - format!( - "{index}:r{}/q{}/b{}/rec{}/v{}", - snapshot.carrier_round, - snapshot.phase_backlog, - snapshot.buffered_authenticated, - snapshot.pending_recovery, - snapshot.clock_valid, - ) - }) - .collect::>() - .join(", "); - table.add_row(row![ - b->"Final/drain state (diagnostic only):", - final_per_validator_state, - ]); - let shadow_input_count = |kind: &str, outcome: &str| { - metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&[kind, outcome]) - .get() - }) - .sum::() - }; - table.add_row(row![ - b->"Carrier transport/ingress:", - format!( - "network sent/disconnected/dropped={}/{}/{}, carriers authenticated/retained/rejected={}/{}/{}, sync requests sent/served={}/{}, sync responses authenticated/rejected={}/{}, subscriptions to/by peers={}..{}/{}..{}", - shadow_input_count("network", "sent"), - shadow_input_count("network", "disconnected"), - shadow_input_count("network", "dropped_backpressure"), - shadow_input_count("carrier", "authenticated"), - shadow_input_count("carrier", "retained_unauthenticated"), - shadow_input_count("carrier", "rejected"), - shadow_input_count("carrier_sync_request", "sent"), - shadow_input_count("carrier_sync_request", "served"), - shadow_input_count("carrier_sync_response", "authenticated"), - shadow_input_count("carrier_sync_response", "rejected"), - metrics - .iter() - .map(|metrics| metrics.subscribed_to_peers.get()) - .min() - .unwrap_or_default(), - metrics - .iter() - .map(|metrics| metrics.subscribed_to_peers.get()) - .max() - .unwrap_or_default(), - metrics - .iter() - .map(|metrics| metrics.subscribed_by_peers.get()) - .min() - .unwrap_or_default(), - metrics - .iter() - .map(|metrics| metrics.subscribed_by_peers.get()) - .max() - .unwrap_or_default(), - ) - ]); - let stage_latency = RBC_DAG_PIPELINE_LATENCY_STAGES - .iter() - .map(|stage| { - let total = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_pipeline_latency_ns_total - .with_label_values(&[stage]) - .get() - }) - .sum::(); - let samples = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[stage]) - .get() - }) - .sum::(); - let maximum = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_pipeline_latency_ns_max - .with_label_values(&[stage]) - .get() - }) - .max() - .unwrap_or_default(); - let average_ms = if samples == 0 { - 0.0 - } else { - total as f64 / samples as f64 / 1_000_000.0 - }; - format!( - "{}={average_ms:.1}/{:.1}ms(n={samples})", - stage.strip_prefix("creation_to_").unwrap_or(stage), - maximum as f64 / 1_000_000.0, - ) - }) - .collect::>() - .join(", "); - table.add_row(row![ - b->"Pipeline latency avg/max:", - stage_latency - ]); - - let commit_distance = |kind: &str| { - let total = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_commit_distance_rounds_total - .with_label_values(&[kind]) - .get() - }) - .sum::(); - let samples = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_commit_distance_samples_total - .with_label_values(&[kind]) - .get() - }) - .sum::(); - let maximum = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_commit_distance_rounds_max - .with_label_values(&[kind]) - .get() - }) - .max() - .unwrap_or_default(); - format_rbc_dag_round_distance(total, samples, maximum) - }; - table.add_row(row![ - b->"Commit distance avg/max:", - format!( - "physical forward={}, backward={}", - commit_distance(RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD), - commit_distance(RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD), - ) - ]); - - let queue_depth = |queue: &str, maximum: bool| { - metrics - .iter() - .map(|metrics| { - if maximum { - metrics - .starfish_rbc_dag_pipeline_queue_depth_max - .with_label_values(&[queue]) - .get() - } else { - metrics - .starfish_rbc_dag_pipeline_queue_depth - .with_label_values(&[queue]) - .get() - } - }) - .max() - .unwrap_or_default() - }; - let highest_projected = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_highest_projected_consensus_round - .get() - }) - .max() - .unwrap_or_default(); - let next_undecided = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_next_undecided_consensus_round - .get() - }) - .min() - .unwrap_or_default(); - let next_undecided_stake = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_next_undecided_projected_stake - .get() - }) - .min() - .unwrap_or_default(); - let last_committed = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_last_committed_consensus_round - .get() - }) - .min() - .unwrap_or_default(); - let hol = RBC_DAG_PROJECTION_HOL_STATES - .iter() - .filter_map(|reason| { - let nodes = metrics - .iter() - .filter(|metrics| { - metrics - .starfish_rbc_dag_projection_hol_state - .with_label_values(&[reason]) - .get() - == 1 - }) - .count(); - (nodes != 0).then(|| format!("{reason}:{nodes}")) - }) - .collect::>() - .join(","); - table.add_row(row![ - b->"Projection/HOL state:", - format!( - "pending local={}/{}, projection={}/{}, highest projected={highest_projected}, oldest undecided={next_undecided} (lag={}, projected stake={next_undecided_stake}), last committed={last_committed} (lag={}), HOL=[{hol}]", - queue_depth("local", false), - queue_depth("local", true), - queue_depth("projection", false), - queue_depth("projection", true), - highest_projected.saturating_sub(next_undecided), - highest_projected.saturating_sub(last_committed), - ) - ]); - let decision_count = |outcome: &str| { - metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&[outcome]) - .get() - }) - .sum::() - }; - table.add_row(row![ - b->"Projection decisions:", - format!( - "direct commit/skip={}/{}, indirect commit/skip={}/{}, undecided={}", - decision_count("direct_commit"), - decision_count("direct_skip"), - decision_count("indirect_commit"), - decision_count("indirect_skip"), - decision_count("undecided"), - ) - ]); - let frontier_count = |stage: &str| { - metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_frontier_events_total - .with_label_values(&[stage]) - .get() - }) - .sum::() - }; - table.add_row(row![ - b->"Frontier event application:", - format!( - "generated={}, applied={}, ignored={}, inflight current/max={}/{}", - frontier_count("generated"), - frontier_count("applied"), - frontier_count("ignored"), - metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_frontiers_inflight.get()) - .sum::(), - metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_frontiers_inflight_max.get()) - .max() - .unwrap_or_default(), - ) - ]); } else if starfish_rbc_dag_shadow_expected { let valid_nodes = metrics .iter() @@ -3029,29 +2215,6 @@ impl MetricReporter { self.block_bundle_size_bytes.lock().clear_receive_all(); self.connection_latency.lock().clear_receive_all(); } - - /// Discard every sample produced before a coordinated benchmark window. - /// `clear_receive_all` intentionally preserves newly received points for - /// periodic reporting; a benchmark reset needs the opposite order. - pub fn reset_for_benchmark_window(&self) { - fn drain_then_clear(histogram: &mut PreciseHistogram) - where - T: Ord + AddAssign + DivUsize + Copy + Default, - { - histogram.receive_all(); - histogram.reset(); - } - - drain_then_clear(&mut self.transaction_committed_latency.lock().histogram); - drain_then_clear(&mut self.block_committed_latency.lock().histogram); - drain_then_clear(&mut self.proposed_block_size_bytes.lock().histogram); - drain_then_clear(&mut self.proposed_header_size_bytes.lock().histogram); - drain_then_clear(&mut self.proposed_transaction_size_bytes.lock().histogram); - drain_then_clear(&mut self.block_bundle_size_bytes.lock().histogram); - for (histogram, _) in &mut self.connection_latency.lock().histograms { - drain_then_clear(histogram); - } - } } pub fn print_network_address_table(addresses: &[SocketAddr]) { @@ -3178,22 +2341,6 @@ mod tests { .starfish_rbc_dag_projection_decisions_total .with_label_values(&["direct_commit"]) .inc(); - metrics.metrics_active.store(true, Ordering::Relaxed); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, - 12, - 2, - 8, - ); - metrics.observe_starfish_rbc_dag_commit_round_distance( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, - 7, - 2, - 5, - ); - metrics.set_starfish_rbc_dag_pipeline_state(2, 3, 9, 7, 3, 6, "awaiting_indirect_anchor"); - metrics.starfish_rbc_dag_frontier_generated(); - metrics.starfish_rbc_dag_frontier_applied(); let gathered = registry.gather(); for name in [ @@ -3218,22 +2365,6 @@ mod tests { "starfish_rbc_dag_shadow_buffered_authenticated", "starfish_rbc_dag_projected_vertices_total", "starfish_rbc_dag_projection_decisions_total", - "starfish_rbc_dag_pipeline_latency_ns_total", - "starfish_rbc_dag_pipeline_latency_samples_total", - "starfish_rbc_dag_pipeline_latency_ns_max", - "starfish_rbc_dag_commit_distance_rounds_total", - "starfish_rbc_dag_commit_distance_samples_total", - "starfish_rbc_dag_commit_distance_rounds_max", - "starfish_rbc_dag_pipeline_queue_depth", - "starfish_rbc_dag_pipeline_queue_depth_max", - "starfish_rbc_dag_highest_projected_consensus_round", - "starfish_rbc_dag_next_undecided_consensus_round", - "starfish_rbc_dag_next_undecided_projected_stake", - "starfish_rbc_dag_last_committed_consensus_round", - "starfish_rbc_dag_projection_hol_state", - "starfish_rbc_dag_frontier_events_total", - "starfish_rbc_dag_frontiers_inflight", - "starfish_rbc_dag_frontiers_inflight_max", ] { assert!( gathered.iter().any(|family| family.get_name() == name), @@ -3242,194 +2373,6 @@ mod tests { } } - #[test] - fn rbc_dag_pipeline_latency_drains_while_protocol_distance_closes_at_cutoff() { - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new(®istry, None, None, None); - - metrics.metrics_active.store(false, Ordering::Relaxed); - metrics - .transaction_metrics_active - .store(false, Ordering::Relaxed); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - 99, - 1, - 99, - ); - metrics.observe_starfish_rbc_dag_commit_round_distance( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, - 99, - 1, - 99, - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) - .get(), - 0 - ); - assert_eq!( - metrics - .starfish_rbc_dag_commit_distance_samples_total - .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD]) - .get(), - 0 - ); - - // The submission window is closed but application observation remains - // open for the bounded drain. Only application pipeline latency may - // advance; protocol round-distance rates stay frozen at cutoff. - metrics - .transaction_metrics_active - .store(true, Ordering::Relaxed); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - 11, - 1, - 11, - ); - metrics.observe_starfish_rbc_dag_commit_round_distance( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, - 99, - 1, - 99, - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) - .get(), - 1 - ); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, - 13, - 1, - 13, - ); - assert_eq!( - metrics.starfish_rbc_dag_frontier_applied_latency_samples(), - 1 - ); - assert_eq!( - metrics.starfish_rbc_dag_frontier_applied_sequenced_transactions(), - 0, - "latency alone must not publish the ordered drain acknowledgement" - ); - metrics.sequenced_transactions_total.inc_by(17); - metrics.starfish_rbc_dag_frontier_applied(); - assert_eq!( - metrics.starfish_rbc_dag_frontier_applied_sequenced_transactions(), - 17, - "frontier application must acknowledge all transactions sequenced before its final latency observation" - ); - assert_eq!( - metrics - .starfish_rbc_dag_commit_distance_samples_total - .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD]) - .get(), - 0 - ); - - metrics.metrics_active.store(true, Ordering::Relaxed); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - 30, - 2, - 20, - ); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - 4, - 1, - 4, - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_latency_ns_total - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) - .get(), - 45 - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) - .get(), - 4 - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_latency_ns_max - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_DELIVERY]) - .get(), - 20 - ); - - metrics.observe_starfish_rbc_dag_commit_round_distance( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, - 7, - 2, - 5, - ); - metrics.observe_starfish_rbc_dag_commit_round_distance( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, - 3, - 1, - 3, - ); - assert_eq!( - metrics - .starfish_rbc_dag_commit_distance_rounds_total - .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD]) - .get(), - 7 - ); - assert_eq!( - metrics - .starfish_rbc_dag_commit_distance_samples_total - .with_label_values(&[RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD]) - .get(), - 1 - ); - assert_eq!( - format_rbc_dag_round_distance(11, 2, 6), - "5.50/6 rounds (n=2)" - ); - - metrics.set_starfish_rbc_dag_pipeline_state(4, 5, 12, 10, 2, 8, "insufficient_lookahead"); - metrics.set_starfish_rbc_dag_pipeline_state(1, 2, 13, 11, 4, 9, "ready"); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_queue_depth_max - .with_label_values(&["local"]) - .get(), - 4 - ); - assert_eq!( - metrics - .starfish_rbc_dag_projection_hol_state - .with_label_values(&["ready"]) - .get(), - 1 - ); - assert_eq!( - metrics - .starfish_rbc_dag_projection_hol_state - .with_label_values(&["insufficient_lookahead"]) - .get(), - 0 - ); - - metrics.starfish_rbc_dag_frontier_generated(); - metrics.starfish_rbc_dag_frontier_generated(); - metrics.starfish_rbc_dag_frontier_applied(); - assert_eq!(metrics.starfish_rbc_dag_frontiers_inflight.get(), 1); - assert_eq!(metrics.starfish_rbc_dag_frontiers_inflight_max.get(), 2); - } - fn autonomous_clock_metrics( round: i64, phase_backlog: i64, @@ -3484,7 +2427,7 @@ mod tests { autonomous_clock_metrics(11, 6, 1), ]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, None, false); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); assert!(summary.verdict_valid); assert_eq!(summary.valid_nodes, 4); @@ -3498,39 +2441,6 @@ mod tests { assert_eq!(summary.maximum_round, 11); } - #[test] - fn autonomous_clock_verdict_uses_cutoff_snapshot_not_drain_state() { - let metrics = vec![ - autonomous_clock_metrics(8, 0, 0), - autonomous_clock_metrics(20, 0, 0), - ]; - let invalid_cutoff = metrics - .iter() - .map(|metrics| metrics.autonomous_clock_benchmark_snapshot()) - .collect::>(); - - // A later drain-time convergence is useful diagnostic state, but it - // cannot repair the measured interval's twelve-round cutoff skew. - metrics[1].starfish_rbc_dag_shadow_carrier_round.set(9); - assert!(summarize_autonomous_clock_benchmark(&metrics, 2, None, None, false).verdict_valid); - assert!( - !summarize_autonomous_clock_benchmark(&metrics, 2, None, Some(&invalid_cutoff), false,) - .verdict_valid - ); - - let valid_cutoff = metrics - .iter() - .map(|metrics| metrics.autonomous_clock_benchmark_snapshot()) - .collect::>(); - metrics[1].starfish_rbc_dag_shadow_clock_valid.set(0); - metrics[1].starfish_rbc_dag_shadow_pending_recovery.set(1); - assert!( - summarize_autonomous_clock_benchmark(&metrics, 2, None, Some(&valid_cutoff), false,) - .verdict_valid, - "drain-time invalidity must remain diagnostic rather than rewriting cutoff validity" - ); - } - #[test] fn autonomous_clock_summary_requires_progress_after_the_benchmark_baseline() { let metrics = vec![ @@ -3543,7 +2453,7 @@ mod tests { .collect::>(); assert!( - !summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), None, false) + !summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) .verdict_valid ); @@ -3577,7 +2487,7 @@ mod tests { } assert!( - summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), None, false) + summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) .verdict_valid ); } @@ -3616,7 +2526,6 @@ mod tests { &[Arc::clone(metrics)], 2, Some(&baselines), - None, true, ) .verdict_valid @@ -3639,7 +2548,6 @@ mod tests { &[Arc::clone(metrics)], 2, Some(&baselines), - None, true, ) .verdict_valid @@ -3658,11 +2566,11 @@ mod tests { let unbounded = autonomous_clock_metrics( 20, STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR * 4 + 1, - starfish_rbc_dag_autonomous_buffered_settled_bound(4) + 1, + STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR * 4 + 1, ); let metrics = vec![no_progress, invalid_clock, unbounded]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, None, false); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); assert!(!summary.verdict_valid); assert_eq!(summary.valid_nodes, 2); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index e170a779..b9133379 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -4,24 +4,22 @@ use std::{ collections::{HashMap, VecDeque}, - fmt, - panic::AssertUnwindSafe, path::PathBuf, sync::{ Arc, - atomic::{AtomicBool, AtomicU32, Ordering}, + atomic::{AtomicU32, Ordering}, }, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::Duration, }; use ahash::{AHashMap, AHashSet}; -use futures::{FutureExt, future::join_all}; +use futures::future::join_all; use rand::seq::SliceRandom; use reed_solomon_simd::ReedSolomonEncoder; use tokio::time::Instant; use tokio::{ select, - sync::{Notify, Semaphore, mpsc, watch}, + sync::{Notify, mpsc}, }; use crate::{ @@ -37,14 +35,10 @@ use crate::{ }, core::Core, core_thread::CoreThreadDispatcher, - crypto::{Blake3Hasher, BlsSigner, MacKey, TransactionsCommitment}, + crypto::{Blake3Hasher, BlsSigner, MacKey}, dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, - metrics::{ - Metrics, RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, - UtilizationTimerVecExt, - }, + metrics::{Metrics, UtilizationTimerVecExt}, network::{BlockBatch, Connection, Network, NetworkMessage, ShardPayload}, runtime::{Handle, JoinError, JoinHandle, sleep}, sailfish_service::{ @@ -53,20 +47,16 @@ use crate::{ shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader, RbcCommitteeId, RbcProtocolInstanceId}, starfish_rbc_dag::{ - CandidateCarrierV1, MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, - RbcDagProtocolInstanceId, projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, + RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, + projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ - CommittedApplicationDiagnosticV1, CommittedFrontierDeltaV1, ShadowAuthorizerV1, - ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, + ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, }, starfish_rbc_dag_shadow_service::{ - ShadowApplicationAuthorizationBasisV1, ShadowServiceErrorV1, ShadowServiceEventV1, - StarfishRbcDagShadowServiceHandleV1, - start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1, - start_starfish_rbc_dag_autonomous_clock_service_paused_with_metrics_v1, - start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1, - start_starfish_rbc_dag_shadow_service_with_metrics_v1, + ShadowServiceErrorV1, ShadowServiceEventV1, StarfishRbcDagShadowServiceHandleV1, + start_starfish_rbc_dag_autonomous_clock_service_v1, + start_starfish_rbc_dag_shadow_service_v1, }, starfish_rbc_service::{ RbcInitialAuthenticator, RbcPhaseAuthorityV1, RbcServiceEvent, RbcServiceHandle, @@ -76,7 +66,7 @@ use crate::{ types::{ AuthorityIndex, AuthoritySet, BlockAuthentication, BlockAuthenticationScheme, BlockDigest, BlockReference, PartialSig, PartialSigKind, ProvableShard, ReconstructedTransactionData, - RoundNumber, TimestampNs, TransactionData, VerifiedBlock, format_authority_index, + RoundNumber, TransactionData, VerifiedBlock, format_authority_index, }, }; @@ -85,525 +75,9 @@ const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); const STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); -const STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); -const STARFISH_RBC_DAG_CORE_CONTROL_CAPACITY: usize = 64; -const STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY: usize = 64; -// Priority entries contain at most one carrier or application payload plus a -// bounded authentication/frame reserve. Proactive carriers may contain both. -// Count and byte accounting are independent so a future wire-size regression -// cannot turn the fixed key caps into an unbounded memory reservoir. -const STARFISH_RBC_DAG_OUTBOUND_FRAME_RESERVE: usize = 128 * 1024; -const STARFISH_RBC_DAG_OUTBOUND_PRIORITY_ENTRY_BYTES: usize = - MAX_CARRIER_CONTENT_SIZE_V1 + STARFISH_RBC_DAG_OUTBOUND_FRAME_RESERVE; -const STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_ENTRY_BYTES: usize = - MAX_CARRIER_CONTENT_SIZE_V1 * 2 + STARFISH_RBC_DAG_OUTBOUND_FRAME_RESERVE; -const STARFISH_RBC_DAG_OUTBOUND_PRIORITY_BYTES: usize = - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY * MAX_CARRIER_CONTENT_SIZE_V1; -const STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_BYTES: usize = - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY * MAX_CARRIER_CONTENT_SIZE_V1; const STARFISH_RBC_DAG_AUTONOMOUS_INSTANCE_CONTEXT: &str = "STARFISH_RBC_DAG_AUTONOMOUS_CLOCK_V1_PROTOCOL_INSTANCE"; -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -enum RbcDagOutboundKeyV1 { - Proactive(BlockReference), - CarrierRequest(BlockReference), - CarrierResponse(BlockReference), - SyncRequest(AuthorityIndex, RoundNumber), - SyncResponse(AuthorityIndex, RoundNumber), - ApplicationPayloadRequest(BlockReference), - ApplicationPayloadResponse(BlockReference), -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum RbcDagOutboundClassV1 { - Priority, - Proactive, -} - -impl RbcDagOutboundClassV1 { - fn label(self) -> &'static str { - match self { - Self::Priority => "priority", - Self::Proactive => "proactive", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum RbcDagOutboundEnqueueV1 { - Added, - Coalesced, -} - -#[derive(Debug)] -enum RbcDagOutboundMailboxErrorV1 { - Unsupported, - InvalidProactive(String), - Serialization(String), - EntryTooLarge { - class: RbcDagOutboundClassV1, - actual: usize, - maximum: usize, - }, - KeyCapacity { - class: RbcDagOutboundClassV1, - capacity: usize, - }, - ByteCapacity { - class: RbcDagOutboundClassV1, - attempted: usize, - capacity: usize, - }, - ConflictingDuplicate { - class: RbcDagOutboundClassV1, - key: RbcDagOutboundKeyV1, - }, - DownstreamSaturated(RbcDagOutboundClassV1), - DownstreamClosed, - Failed(String), -} - -impl fmt::Display for RbcDagOutboundMailboxErrorV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Unsupported => write!(formatter, "unsupported non-RBC-DAG outbound message"), - Self::InvalidProactive(error) => { - write!(formatter, "invalid proactive carrier: {error}") - } - Self::Serialization(error) => { - write!(formatter, "outbound message serialization failed: {error}") - } - Self::EntryTooLarge { - class, - actual, - maximum, - } => write!( - formatter, - "{} outbound entry is {actual} bytes, maximum {maximum}", - class.label(), - ), - Self::KeyCapacity { class, capacity } => write!( - formatter, - "{} outbound key capacity {capacity} exhausted", - class.label(), - ), - Self::ByteCapacity { - class, - attempted, - capacity, - } => write!( - formatter, - "{} outbound byte capacity {capacity} exhausted by {attempted} bytes", - class.label(), - ), - Self::ConflictingDuplicate { class, key } => write!( - formatter, - "conflicting {} outbound duplicate for {key:?}", - class.label(), - ), - Self::DownstreamSaturated(class) => { - write!( - formatter, - "downstream {} network channel saturated", - class.label() - ) - } - Self::DownstreamClosed => write!(formatter, "downstream network sender closed"), - Self::Failed(reason) => write!(formatter, "outbound mailbox already failed: {reason}"), - } - } -} - -struct RbcDagOutboundEntryV1 { - message: NetworkMessage, - framed_bytes: usize, -} - -struct RbcDagOutboundLaneV1 { - order: VecDeque, - entries: AHashMap, - bytes: usize, - key_capacity: usize, - byte_capacity: usize, - entry_byte_capacity: usize, -} - -impl RbcDagOutboundLaneV1 { - fn new(key_capacity: usize, byte_capacity: usize, entry_byte_capacity: usize) -> Self { - Self { - order: VecDeque::new(), - entries: AHashMap::new(), - bytes: 0, - key_capacity, - byte_capacity, - entry_byte_capacity, - } - } - - fn enqueue( - &mut self, - class: RbcDagOutboundClassV1, - key: RbcDagOutboundKeyV1, - entry: RbcDagOutboundEntryV1, - ) -> Result { - if let Some(existing) = self.entries.get(&key) { - return if rbc_dag_outbound_messages_equal(&existing.message, &entry.message) { - Ok(RbcDagOutboundEnqueueV1::Coalesced) - } else { - Err(RbcDagOutboundMailboxErrorV1::ConflictingDuplicate { class, key }) - }; - } - if entry.framed_bytes > self.entry_byte_capacity { - return Err(RbcDagOutboundMailboxErrorV1::EntryTooLarge { - class, - actual: entry.framed_bytes, - maximum: self.entry_byte_capacity, - }); - } - if self.entries.len() >= self.key_capacity { - return Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { - class, - capacity: self.key_capacity, - }); - } - let attempted = self.bytes.checked_add(entry.framed_bytes).ok_or( - RbcDagOutboundMailboxErrorV1::ByteCapacity { - class, - attempted: usize::MAX, - capacity: self.byte_capacity, - }, - )?; - if attempted > self.byte_capacity { - return Err(RbcDagOutboundMailboxErrorV1::ByteCapacity { - class, - attempted, - capacity: self.byte_capacity, - }); - } - self.bytes = attempted; - self.order.push_back(key); - assert!(self.entries.insert(key, entry).is_none()); - Ok(RbcDagOutboundEnqueueV1::Added) - } - - fn pop_front(&mut self) -> Option { - let key = self.order.pop_front()?; - let entry = self - .entries - .remove(&key) - .expect("queued RBC-DAG outbound key retains its exact entry"); - self.bytes = self - .bytes - .checked_sub(entry.framed_bytes) - .expect("RBC-DAG outbound byte accounting cannot underflow"); - Some(entry.message) - } -} - -struct RbcDagOutboundMailboxStateV1 { - priority: RbcDagOutboundLaneV1, - proactive: RbcDagOutboundLaneV1, - failure: Option, -} - -impl RbcDagOutboundMailboxStateV1 { - fn production() -> Self { - Self { - priority: RbcDagOutboundLaneV1::new( - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - STARFISH_RBC_DAG_OUTBOUND_PRIORITY_BYTES, - STARFISH_RBC_DAG_OUTBOUND_PRIORITY_ENTRY_BYTES, - ), - proactive: RbcDagOutboundLaneV1::new( - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_BYTES, - STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_ENTRY_BYTES, - ), - failure: None, - } - } -} - -struct RbcDagOutboundMailboxInnerV1 { - state: parking_lot::Mutex, - notified: Notify, -} - -#[derive(Clone)] -struct RbcDagOutboundMailboxV1 { - inner: Arc, -} - -impl RbcDagOutboundMailboxV1 { - fn new() -> Self { - Self::from_state(RbcDagOutboundMailboxStateV1::production()) - } - - fn from_state(state: RbcDagOutboundMailboxStateV1) -> Self { - Self { - inner: Arc::new(RbcDagOutboundMailboxInnerV1 { - state: parking_lot::Mutex::new(state), - notified: Notify::new(), - }), - } - } - - fn enqueue( - &self, - message: NetworkMessage, - committee: &Committee, - ) -> Result { - if let Some(reason) = self.inner.state.lock().failure.clone() { - return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); - } - let (class, key) = match rbc_dag_outbound_classification(&message, committee) { - Ok(classification) => classification, - Err(error) => { - self.fail(&error); - return Err(error); - } - }; - let framed_bytes = match bincode::serialized_size(&message) - .map_err(|error| RbcDagOutboundMailboxErrorV1::Serialization(error.to_string())) - .and_then(|size| { - usize::try_from(size) - .ok() - .and_then(|size| size.checked_add(4)) - .ok_or_else(|| { - RbcDagOutboundMailboxErrorV1::Serialization( - "framed size does not fit usize".to_owned(), - ) - }) - }) { - Ok(framed_bytes) => framed_bytes, - Err(error) => { - self.fail(&error); - return Err(error); - } - }; - let entry = RbcDagOutboundEntryV1 { - message, - framed_bytes, - }; - let mut state = self.inner.state.lock(); - if let Some(reason) = state.failure.clone() { - return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); - } - let result = match class { - RbcDagOutboundClassV1::Priority => state.priority.enqueue(class, key, entry), - RbcDagOutboundClassV1::Proactive => state.proactive.enqueue(class, key, entry), - }; - match result { - Ok(outcome) => { - drop(state); - if outcome == RbcDagOutboundEnqueueV1::Added { - self.inner.notified.notify_one(); - } - Ok(outcome) - } - Err(error) => { - state.failure = Some(error.to_string()); - drop(state); - self.inner.notified.notify_waiters(); - Err(error) - } - } - } - - fn fail(&self, error: &RbcDagOutboundMailboxErrorV1) { - let mut state = self.inner.state.lock(); - state.failure.get_or_insert_with(|| error.to_string()); - drop(state); - self.inner.notified.notify_waiters(); - } - - fn try_pop(&self) -> Option<(RbcDagOutboundClassV1, NetworkMessage)> { - let mut state = self.inner.state.lock(); - if state.failure.is_some() { - return None; - } - state - .priority - .pop_front() - .map(|message| (RbcDagOutboundClassV1::Priority, message)) - .or_else(|| { - state - .proactive - .pop_front() - .map(|message| (RbcDagOutboundClassV1::Proactive, message)) - }) - } - - async fn recv(&self) -> Option<(RbcDagOutboundClassV1, NetworkMessage)> { - loop { - let notified = self.inner.notified.notified(); - if let Some(message) = self.try_pop() { - return Some(message); - } - if self.inner.state.lock().failure.is_some() { - return None; - } - notified.await; - } - } -} - -fn rbc_dag_outbound_classification( - message: &NetworkMessage, - committee: &Committee, -) -> Result<(RbcDagOutboundClassV1, RbcDagOutboundKeyV1), RbcDagOutboundMailboxErrorV1> { - let priority = RbcDagOutboundClassV1::Priority; - match message { - NetworkMessage::RbcDagShadowCarrier(carrier) => { - let candidate = - CandidateCarrierV1::decode_wire(&carrier.canonical_carrier, committee, None) - .map_err(|error| { - RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) - })?; - let canonical = candidate.canonical_wire_bytes().map_err(|error| { - RbcDagOutboundMailboxErrorV1::InvalidProactive(error.to_string()) - })?; - if canonical != carrier.canonical_carrier { - return Err(RbcDagOutboundMailboxErrorV1::InvalidProactive( - "non-canonical carrier wire".to_owned(), - )); - } - Ok(( - RbcDagOutboundClassV1::Proactive, - RbcDagOutboundKeyV1::Proactive(candidate.reference()), - )) - } - NetworkMessage::RbcDagShadowCarrierRequest(reference) => { - Ok((priority, RbcDagOutboundKeyV1::CarrierRequest(*reference))) - } - NetworkMessage::RbcDagShadowCarrierResponse(response) => Ok(( - priority, - RbcDagOutboundKeyV1::CarrierResponse(response.reference), - )), - NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => Ok(( - priority, - RbcDagOutboundKeyV1::SyncRequest(request.author, request.round), - )), - NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => Ok(( - priority, - RbcDagOutboundKeyV1::SyncResponse(response.author, response.round), - )), - NetworkMessage::RbcDagApplicationPayloadRequest(application) => Ok(( - priority, - RbcDagOutboundKeyV1::ApplicationPayloadRequest(*application), - )), - NetworkMessage::RbcDagApplicationPayloadResponse(response) => Ok(( - priority, - RbcDagOutboundKeyV1::ApplicationPayloadResponse(response.application), - )), - _ => Err(RbcDagOutboundMailboxErrorV1::Unsupported), - } -} - -fn rbc_dag_outbound_messages_equal(left: &NetworkMessage, right: &NetworkMessage) -> bool { - match (left, right) { - (NetworkMessage::RbcDagShadowCarrier(left), NetworkMessage::RbcDagShadowCarrier(right)) => { - left == right - } - ( - NetworkMessage::RbcDagShadowCarrierRequest(left), - NetworkMessage::RbcDagShadowCarrierRequest(right), - ) => left == right, - ( - NetworkMessage::RbcDagShadowCarrierResponse(left), - NetworkMessage::RbcDagShadowCarrierResponse(right), - ) => left == right, - ( - NetworkMessage::RbcDagShadowCarrierSyncRequest(left), - NetworkMessage::RbcDagShadowCarrierSyncRequest(right), - ) => left == right, - ( - NetworkMessage::RbcDagShadowCarrierSyncResponse(left), - NetworkMessage::RbcDagShadowCarrierSyncResponse(right), - ) => left == right, - ( - NetworkMessage::RbcDagApplicationPayloadRequest(left), - NetworkMessage::RbcDagApplicationPayloadRequest(right), - ) => left == right, - ( - NetworkMessage::RbcDagApplicationPayloadResponse(left), - NetworkMessage::RbcDagApplicationPayloadResponse(right), - ) => { - left.application == right.application - && left.transaction_data.transactions() == right.transaction_data.transactions() - } - _ => false, - } -} - -fn current_timestamp_ns() -> TimestampNs { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - .try_into() - .unwrap_or(TimestampNs::MAX) -} - -fn latency_since_timestamps( - creation_times: impl IntoIterator, - now_ns: TimestampNs, -) -> (u64, u64, u64) { - creation_times.into_iter().fold( - (0u64, 0u64, 0u64), - |(total, samples, maximum), creation_time| { - let latency = now_ns.saturating_sub(creation_time); - ( - total.saturating_add(latency), - samples.saturating_add(1), - maximum.max(latency), - ) - }, - ) -} - -#[derive(Default)] -struct CommitRoundDistanceBatch { - physical_forward: (u64, u64, u64), - physical_backward: (u64, u64, u64), -} - -impl CommitRoundDistanceBatch { - fn from_diagnostics( - diagnostics: impl IntoIterator, - ) -> Self { - let mut batch = Self::default(); - for diagnostic in diagnostics { - let aggregate = if diagnostic.physical_carrier_round_delta >= 0 { - &mut batch.physical_forward - } else { - &mut batch.physical_backward - }; - let value = diagnostic.physical_carrier_round_delta.unsigned_abs(); - aggregate.0 = aggregate.0.saturating_add(value); - aggregate.1 = aggregate.1.saturating_add(1); - aggregate.2 = aggregate.2.max(value); - } - batch - } - - fn observe(self, metrics: &Metrics) { - for (kind, (total, samples, maximum)) in [ - ( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_FORWARD, - self.physical_forward, - ), - ( - RBC_DAG_COMMIT_DISTANCE_PHYSICAL_BACKWARD, - self.physical_backward, - ), - ] { - metrics.observe_starfish_rbc_dag_commit_round_distance(kind, total, samples, maximum); - } - } -} - /// Recover the exact locally selected Starfish-RBC chain so the persisted /// non-authoritative shadow can reconcile a WAL that ended before the direct /// DAG. The newest local block determines the branch when a Byzantine test @@ -673,12 +147,10 @@ fn recovered_local_rbc_headers( } fn shadow_transport_error_invalidates_run(error: &ShadowServiceErrorV1) -> bool { - match error { - ShadowServiceErrorV1::Stopped => true, - #[cfg(test)] - ShadowServiceErrorV1::Overloaded { .. } => true, - _ => false, - } + matches!( + error, + ShadowServiceErrorV1::Overloaded { .. } | ShadowServiceErrorV1::Stopped + ) } fn invalidate_shadow_run(metrics: &Metrics) { @@ -839,49 +311,6 @@ async fn send_network_message_reliably( } } -async fn run_rbc_dag_outbound_worker( - mailbox: RbcDagOutboundMailboxV1, - proactive_sender: mpsc::Sender, - priority_sender: mpsc::Sender, - mut outbound_failure: watch::Receiver>, -) -> Result<(), RbcDagOutboundMailboxErrorV1> { - loop { - if let Some(reason) = outbound_failure.borrow().clone() { - return Err(RbcDagOutboundMailboxErrorV1::Failed(reason)); - } - let next = tokio::select! { - biased; - changed = outbound_failure.changed() => { - if changed.is_err() { - return Err(RbcDagOutboundMailboxErrorV1::DownstreamClosed); - } - continue; - } - next = mailbox.recv() => next, - }; - let Some((class, message)) = next else { - let state = mailbox.inner.state.lock(); - return match &state.failure { - Some(reason) => Err(RbcDagOutboundMailboxErrorV1::Failed(reason.clone())), - None => Ok(()), - }; - }; - let result = match class { - RbcDagOutboundClassV1::Priority => priority_sender.try_send(message), - RbcDagOutboundClassV1::Proactive => proactive_sender.try_send(message), - }; - match result { - Ok(()) => {} - Err(mpsc::error::TrySendError::Full(_)) => { - return Err(RbcDagOutboundMailboxErrorV1::DownstreamSaturated(class)); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - return Err(RbcDagOutboundMailboxErrorV1::DownstreamClosed); - } - } - } -} - async fn broadcast_sailfish_cert_messages( senders: &[mpsc::Sender], cert_messages: &[crate::types::CertMessage], @@ -1533,16 +962,14 @@ impl ConnectionHandler ConnectionHandler { - if self.inner.embedded_rbc_authority { - tracing::warn!( - peer = self.peer_id, - "Rejected generic block batch while embedded RBC-DAG authority is active" - ); - } else { - self.handle_batch(*blocks).await; - } + self.handle_batch(*blocks).await; } NetworkMessage::MissingParentsRequest(refs) => { - if self.inner.embedded_rbc_authority { - tracing::debug!( - peer = self.peer_id, - count = refs.len(), - "Ignored legacy missing-parent request in standalone RBC-DAG mode" - ); - return true; - } return self.handle_missing_parents_request(refs).await; } NetworkMessage::MissingTxDataRequest(refs) => { - if self.inner.embedded_rbc_authority { - tracing::debug!( - peer = self.peer_id, - count = refs.len(), - "Ignored legacy transaction-data request in standalone RBC-DAG mode" - ); - return true; - } return self.handle_missing_tx_data_request(refs).await; } NetworkMessage::PartialSig(sig) => { @@ -1704,7 +1108,7 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_reliably(self.peer_id, envelope).await { + if let Err(error) = shadow.carrier(self.peer_id, envelope) { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1714,10 +1118,7 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .carrier_request_reliably(self.peer_id, reference) - .await - { + if let Err(error) = shadow.carrier_request(self.peer_id, reference) { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1727,10 +1128,7 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .carrier_response_reliably(self.peer_id, response) - .await - { + if let Err(error) = shadow.carrier_response(self.peer_id, response) { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1740,10 +1138,7 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .carrier_sync_request_reliably(self.peer_id, request) - .await - { + if let Err(error) = shadow.carrier_sync_request(self.peer_id, request) { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1755,10 +1150,7 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .carrier_sync_response_reliably(self.peer_id, response) - .await - { + if let Err(error) = shadow.carrier_sync_response(self.peer_id, response) { if shadow_transport_error_invalidates_run(&error) { invalidate_shadow_run(&self.metrics); } @@ -1768,39 +1160,6 @@ impl ConnectionHandler { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .application_payload_request_reliably(self.peer_id, application) - .await - { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); - } - tracing::warn!( - ?application, - peer = self.peer_id, - "Failed to forward RBC-DAG application-payload request: {error}" - ); - } - } - } - NetworkMessage::RbcDagApplicationPayloadResponse(response) => { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow - .application_payload_response_reliably(self.peer_id, response) - .await - { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); - } - tracing::warn!( - peer = self.peer_id, - "Failed to forward RBC-DAG application-payload response: {error}" - ); - } - } - } } true } @@ -1813,11 +1172,6 @@ impl ConnectionHandler { rbc_event_task: Option>, rbc_service_task: Option>, rbc_dag_shadow_event_task: Option>, - rbc_dag_core_control_task: Option>, - rbc_dag_assignment_task: Option>, rbc_dag_shadow_service_task: Option>, - rbc_dag_clock_bridge_state: Option>, cordial_knowledge_task: JoinHandle<()>, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum RbcDagClockBridgeStateV1 { - Pending, - Failed, - Activated, -} - -type RbcDagPayloadVerificationTaskV1 = - JoinHandle, tokio::task::JoinError>>; - -enum RbcDagAuthorizedPayloadV1 { - None, - AlreadyAvailable(Option>), - Verify(RbcDagPayloadVerificationTaskV1), -} - -enum RbcDagCoreControlCommandV1 { - AuthorizedApplicationObserved { - carrier: BlockReference, - header: RbcCanonicalHeader, - authorization_basis: ShadowApplicationAuthorizationBasisV1, - payload: RbcDagAuthorizedPayloadV1, - }, - Frontier(CommittedFrontierDeltaV1), - /// Every recovery application and frontier preceding this marker must be - /// fully applied before activation is legal. - Ready, - Activate, - /// Intentional end-of-stream marker. A sender disappearing without this - /// marker is an authoritative runtime failure. - Drain, -} - -enum RbcDagApplicationAssignmentCommandV1 { - Assigned(BlockReference), - Drain, -} - -#[derive(Default)] -struct RbcDagCoreControlGateV1 { - startup_ready: bool, - clean_drain: bool, -} - -impl RbcDagCoreControlGateV1 { - fn record_ready(&mut self) { - self.startup_ready = true; - } - - fn activation_allowed(&self) -> bool { - self.startup_ready - } - - fn record_clean_drain(&mut self) { - self.clean_drain = true; - } -} - -async fn wait_for_rbc_dag_clock_bridge_activation( - bridge_state: &mut watch::Receiver, -) -> Result<(), String> { - loop { - match *bridge_state.borrow_and_update() { - RbcDagClockBridgeStateV1::Pending => {} - RbcDagClockBridgeStateV1::Failed => { - return Err( - "RBC-DAG startup recovery was invalid before Core clock activation".to_string(), - ); - } - RbcDagClockBridgeStateV1::Activated => return Ok(()), - } - bridge_state - .changed() - .await - .map_err(|_| "RBC-DAG event bridge stopped before Core clock activation".to_string())?; - } -} - -fn fail_rbc_dag_clock_bridge(bridge_tx: Option<&watch::Sender>) { - if let Some(bridge_tx) = bridge_tx { - // Failure is permanently terminal, including after a caller already - // observed activation. No later authority command may reach Core. - bridge_tx.send_if_modified(|state| { - if *state != RbcDagClockBridgeStateV1::Failed { - *state = RbcDagClockBridgeStateV1::Failed; - true - } else { - false - } - }); - } -} - -fn rbc_dag_clock_bridge_failed( - bridge_tx: Option<&watch::Sender>, -) -> bool { - bridge_tx.is_some_and(|bridge_tx| *bridge_tx.borrow() == RbcDagClockBridgeStateV1::Failed) -} - pub(crate) struct NetworkSyncSignals { block_ready_notify: Arc, proposal_round_notify: Arc, @@ -2522,21 +1774,11 @@ pub struct NetworkSyncerInner { /// another peer or the actor's local HeaderStaged/Delivered effects. rbc_peer_senders: parking_lot::RwLock>>, - /// Bounded, keyed RBC-DAG transport mailboxes. Exact repair is admitted - /// independently of proactive carrier fan-out and drains first. - rbc_dag_peer_mailboxes: parking_lot::RwLock>, pub leader_timeout: Duration, pub soft_block_timeout: Duration, - metrics: Arc, - rbc_dag_clock_bridge_tx: Option>, - rbc_dag_shutdown_started: Arc, /// Sailfish++ service handle for sending control messages /// (timeout/no-vote). None for non-SailfishPlusPlus protocols. pub sailfish_handle: Option, - /// When true, only typed carrier-authorized application ingress may reach - /// the core. Peer-controlled block batches and legacy data-recovery paths - /// are rejected even if their serialized `DataSource` claims authority. - pub embedded_rbc_authority: bool, /// Central Starfish-RBC service. Connection workers only forward their /// trusted peer identity and wire payload into this single owner. pub(crate) starfish_rbc_service: Option, @@ -2549,509 +1791,6 @@ pub struct NetworkSyncerInner { pub start_time: std::time::Instant, } -struct RbcDagAppliedFrontierObservationV1 { - carrier_count: u64, - application_count: u64, - application_creation_times: Vec, - commit_round_distances: CommitRoundDistanceBatch, -} - -trait RbcDagCoreControlTargetV1: Send + Sync { - async fn stage_authorized_application( - &self, - header: RbcCanonicalHeader, - authorization_basis: ShadowApplicationAuthorizationBasisV1, - ) -> Result<(), String>; - - fn restore_available_application( - &self, - application: BlockReference, - payload: Option>, - ) -> Result<(), String>; - - async fn materialize_authorized_payload( - &self, - item: ReconstructedTransactionData, - ) -> Result<(), String>; - - async fn apply_frontier(&self, delta: CommittedFrontierDeltaV1) -> Result; - - async fn activate_authority(&self) -> Result<(), String>; - - async fn apply_assignment(&self, reference: BlockReference) -> Result<(), String>; -} - -impl RbcDagCoreControlTargetV1 - for NetworkSyncerInner -{ - async fn stage_authorized_application( - &self, - header: RbcCanonicalHeader, - authorization_basis: ShadowApplicationAuthorizationBasisV1, - ) -> Result<(), String> { - let block_ref = header.reference(); - let (missing_parents, _) = self.syncer.add_authorized_rbc_dag_header(header).await; - self.cordial_knowledge - .send(CordialKnowledgeMessage::DagParts { - headers: vec![block_ref], - shards: Vec::new(), - }); - if !missing_parents.is_empty() { - tracing::debug!( - ?block_ref, - ?missing_parents, - ?authorization_basis, - "Authorized RBC-DAG application waits for parent materialization" - ); - } - Ok(()) - } - - fn restore_available_application( - &self, - application: BlockReference, - payload: Option>, - ) -> Result<(), String> { - // The actor is intentionally stopped before the authoritative FIFO is - // drained. Core already owns this data-available block, and restart - // recovery deterministically rehydrates the corresponding shadow - // state, so no actor callback is required while shutting down. - if self.rbc_dag_shutdown_started.load(Ordering::Acquire) { - return Ok(()); - } - let shadow = self - .starfish_rbc_dag_shadow_service - .as_ref() - .ok_or_else(|| "authoritative payload callback target is unavailable".to_owned())?; - if let Some(payload) = payload { - shadow - .verified_application_payload(application, payload) - .map_err(|error| error.to_string())?; - } - shadow - .application_data_available(application) - .map_err(|error| error.to_string()) - } - - async fn materialize_authorized_payload( - &self, - item: ReconstructedTransactionData, - ) -> Result<(), String> { - let block_ref = item.block_reference; - self.cordial_knowledge - .send(CordialKnowledgeMessage::DagParts { - headers: Vec::new(), - shards: vec![block_ref], - }); - if let Some(shard_tx) = self.shard_tx.lock().as_ref() { - let _ = shard_tx.send(vec![ShardMessage::FullBlock(block_ref)]); - } - let verified_payload = Arc::new(item.transaction_data.clone()); - self.syncer.add_authorized_rbc_dag_payload(item).await; - // Core materialization is durable authority work and must complete - // before a queued frontier. The actor callback is ephemeral and the - // actor has already been stopped by the graceful shutdown protocol. - if self.rbc_dag_shutdown_started.load(Ordering::Acquire) { - return Ok(()); - } - self.starfish_rbc_dag_shadow_service - .as_ref() - .ok_or_else(|| "authoritative payload callback target is unavailable".to_owned())? - .verified_application_payload(block_ref, verified_payload) - .map_err(|error| error.to_string()) - } - - async fn apply_frontier(&self, delta: CommittedFrontierDeltaV1) -> Result { - self.syncer - .apply_starfish_rbc_dag_frontier(delta) - .await - .map_err(|error| error.to_string()) - } - - async fn activate_authority(&self) -> Result<(), String> { - self.syncer.activate_starfish_rbc_dag_authority().await; - Ok(()) - } - - async fn apply_assignment(&self, reference: BlockReference) -> Result<(), String> { - self.syncer - .apply_starfish_rbc_dag_application_assigned(reference) - .await; - Ok(()) - } -} - -impl RbcDagAppliedFrontierObservationV1 { - fn from_delta(delta: &CommittedFrontierDeltaV1) -> Self { - Self { - carrier_count: delta.carriers.len() as u64, - application_count: delta.applications.len() as u64, - application_creation_times: delta - .applications - .iter() - .map(RbcCanonicalHeader::meta_creation_time_ns) - .collect(), - commit_round_distances: CommitRoundDistanceBatch::from_diagnostics( - delta.application_diagnostics.iter().copied(), - ), - } - } - - fn observe(self, metrics: &Metrics) { - let (total_ns, samples, max_ns) = - latency_since_timestamps(self.application_creation_times, current_timestamp_ns()); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_FRONTIER_APPLIED, - total_ns, - samples, - max_ns, - ); - self.commit_round_distances.observe(metrics); - metrics.starfish_rbc_dag_frontier_applied(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "carrier"]) - .inc_by(self.carrier_count); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .inc_by(self.application_count); - } -} - -fn fail_rbc_dag_authority( - metrics: &Metrics, - bridge_tx: Option<&watch::Sender>, - reason: &str, -) { - invalidate_shadow_run(metrics); - fail_rbc_dag_clock_bridge(bridge_tx); - tracing::error!( - reason, - "RBC-DAG authoritative Core-control worker failed closed" - ); -} - -fn fail_rbc_dag_outbound_transport( - metrics: &Metrics, - bridge_tx: Option<&watch::Sender>, - embedded_rbc_authority: bool, - recipient: AuthorityIndex, - error: &RbcDagOutboundMailboxErrorV1, -) { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "overloaded"]) - .inc(); - if embedded_rbc_authority { - fail_rbc_dag_authority( - metrics, - bridge_tx, - &format!("RBC-DAG outbound mailbox for authority {recipient} failed: {error}"), - ); - } else { - invalidate_shadow_run(metrics); - tracing::error!( - peer = recipient, - ?error, - "RBC-DAG observational outbound mailbox failed" - ); - } -} - -struct RbcDagEventRouterGuardV1 { - metrics: Arc, - bridge_tx: watch::Sender, - clean_exit: bool, -} - -impl RbcDagEventRouterGuardV1 { - fn new(metrics: Arc, bridge_tx: watch::Sender) -> Self { - Self { - metrics, - bridge_tx, - clean_exit: false, - } - } - - fn record_clean_exit(&mut self) { - self.clean_exit = true; - } -} - -impl Drop for RbcDagEventRouterGuardV1 { - fn drop(&mut self) { - if !self.clean_exit { - fail_rbc_dag_authority( - &self.metrics, - Some(&self.bridge_tx), - "RBC-DAG authoritative event router stopped unexpectedly", - ); - } - } -} - -async fn drain_rbc_dag_authorized_payload(payload: RbcDagAuthorizedPayloadV1) { - if let RbcDagAuthorizedPayloadV1::Verify(payload_verification) = payload { - // Await rather than merely drop the JoinHandle: dropping detaches the - // verifier and could let Reed-Solomon work outlive Core shutdown. - let _ = payload_verification.await; - } -} - -async fn drain_rbc_dag_control_command_payload(command: RbcDagCoreControlCommandV1) { - if let RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { payload, .. } = command { - drain_rbc_dag_authorized_payload(payload).await; - } -} - -async fn enqueue_rbc_dag_core_control( - sender: &mpsc::Sender, - command: RbcDagCoreControlCommandV1, - metrics: &Metrics, - bridge_tx: Option<&watch::Sender>, -) -> bool { - let drain = matches!(command, RbcDagCoreControlCommandV1::Drain); - if !drain && rbc_dag_clock_bridge_failed(bridge_tx) { - drain_rbc_dag_control_command_payload(command).await; - return false; - } - match sender.send(command).await { - Ok(()) => return true, - Err(error) => drain_rbc_dag_control_command_payload(error.0).await, - } - fail_rbc_dag_authority( - metrics, - bridge_tx, - "bounded RBC-DAG Core-control channel closed unexpectedly", - ); - false -} - -async fn run_rbc_dag_core_control_worker( - target: Arc, - metrics: Arc, - bridge_tx: watch::Sender, - shutdown_started: Arc, - mut commands: mpsc::Receiver, -) { - let mut gate = RbcDagCoreControlGateV1::default(); - while let Some(command) = commands.recv().await { - if matches!(command, RbcDagCoreControlCommandV1::Drain) { - gate.record_clean_drain(); - break; - } - if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { - drain_rbc_dag_control_command_payload(command).await; - continue; - } - if !shutdown_started.load(Ordering::Acquire) - && metrics.starfish_rbc_dag_shadow_clock_valid.get() == 0 - { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - "authoritative runtime validity was revoked", - ); - drain_rbc_dag_control_command_payload(command).await; - continue; - } - - match command { - RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier, - header, - authorization_basis, - payload, - } => { - let block_ref = header.reference(); - if let Err(error) = target - .stage_authorized_application(header, authorization_basis) - .await - { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!("authorized header insertion failed for {block_ref}: {error}"), - ); - drain_rbc_dag_authorized_payload(payload).await; - continue; - } - - let item = match payload { - RbcDagAuthorizedPayloadV1::None => continue, - RbcDagAuthorizedPayloadV1::AlreadyAvailable(payload) => { - if !shutdown_started.load(Ordering::Acquire) { - if let Err(error) = - target.restore_available_application(block_ref, payload) - { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!( - "materialized payload callback failed for {block_ref}: {error}" - ), - ); - } - } - continue; - } - RbcDagAuthorizedPayloadV1::Verify(payload_verification) => { - match payload_verification.await { - Ok(Ok(Ok(item))) => item, - // Payload bytes remain untrusted even when their - // enclosing header is authorized. Reject only this - // observation so another holder can recover it. - Ok(Ok(Err(error))) => { - tracing::warn!( - ?carrier, - ?block_ref, - ?authorization_basis, - ?error, - "Rejected RBC-DAG application payload" - ); - continue; - } - Ok(Err(error)) | Err(error) - if shutdown_started.load(Ordering::Acquire) => - { - tracing::debug!( - ?block_ref, - ?error, - "RBC-DAG payload verifier stopped during shutdown" - ); - continue; - } - Ok(Err(error)) | Err(error) => { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!( - "authorized payload verifier stopped for {block_ref} in carrier {carrier}: {error}" - ), - ); - continue; - } - } - } - }; - if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { - continue; - } - if let Err(error) = target.materialize_authorized_payload(item).await { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!("verified payload callback failed for {block_ref}: {error}"), - ); - } - } - RbcDagCoreControlCommandV1::Frontier(delta) => { - let observation = RbcDagAppliedFrontierObservationV1::from_delta(&delta); - match target.apply_frontier(delta).await { - Ok(true) => observation.observe(&metrics), - // Exact replay is an idempotent Core acknowledgment. It - // must not double-count application or frontier effects. - Ok(false) => {} - Err(error) => fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!("authoritative frontier was rejected: {error}"), - ), - } - } - RbcDagCoreControlCommandV1::Ready => gate.record_ready(), - RbcDagCoreControlCommandV1::Activate => { - if !gate.activation_allowed() { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - "clock activation arrived before the startup recovery barrier", - ); - continue; - } - if let Err(error) = target.activate_authority().await { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!("Core authority activation failed: {error}"), - ); - continue; - } - if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { - continue; - } - metrics.starfish_rbc_dag_shadow_clock_valid.set(1); - bridge_tx.send_if_modified(|state| { - if *state == RbcDagClockBridgeStateV1::Pending { - *state = RbcDagClockBridgeStateV1::Activated; - true - } else { - false - } - }); - } - RbcDagCoreControlCommandV1::Drain => unreachable!("drain handled above"), - } - } - - if !gate.clean_drain { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - "RBC-DAG Core-control sender disappeared before a graceful drain", - ); - } -} - -async fn run_rbc_dag_application_assignment_worker( - target: Arc, - metrics: Arc, - bridge_tx: watch::Sender, - shutdown_started: Arc, - mut assignments: mpsc::Receiver, -) { - let mut clean_drain = false; - while let Some(command) = assignments.recv().await { - match command { - RbcDagApplicationAssignmentCommandV1::Drain => { - clean_drain = true; - break; - } - RbcDagApplicationAssignmentCommandV1::Assigned(reference) => { - // Assignments release an ephemeral producer gate. The local - // application was already durably fixed before this event, so - // creating a fresh Core block while the shadow actor is - // stopped is both unnecessary and unsafe. - if shutdown_started.load(Ordering::Acquire) { - continue; - } - if rbc_dag_clock_bridge_failed(Some(&bridge_tx)) { - continue; - } - if let Err(error) = target.apply_assignment(reference).await { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - &format!("Core application-assignment acknowledgment failed: {error}"), - ); - } - } - } - } - if !clean_drain && !shutdown_started.load(Ordering::Acquire) { - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - "RBC-DAG assignment sender disappeared unexpectedly", - ); - } -} - impl NetworkSyncer { pub async fn start( network: Network, @@ -3063,29 +1802,18 @@ impl NetworkSyncer partial_sig_outbox_rx: Option>, bls_cert_aggregator: Option, bls_signer: Option, - rbc_dag_clock_start_paused: bool, ) -> Self { let handle = Handle::current(); let block_ready_notify = Arc::new(Notify::new()); let proposal_round_notify = Arc::new(Notify::new()); - let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; - let (committed, committed_leaders_count) = - core.take_recovered_committed(embedded_rbc_authority); + let (committed, committed_leaders_count) = core.take_recovered_committed(); commit_observer.recover_committed(committed, committed_leaders_count); let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); - let rbc_dag_frontier_recovery_cursor = embedded_rbc_authority - .then(|| core.rbc_dag_frontier_recovery_cursor()) - .flatten(); let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { match recovered_local_rbc_headers(&core) { Ok(headers) => Some(headers), - Err(error) if embedded_rbc_authority => { - panic!( - "embedded RBC-DAG authority cannot start without reconcilable local history: {error}" - ); - } Err(error) => { // A partial local history would make delivery comparisons // meaningless in mirror mode and make autonomous local @@ -3130,7 +1858,7 @@ impl NetworkSyncer .as_ref() .map(|tx| SailfishServiceHandle::new(tx.clone())); let (starfish_rbc_service, rbc_event_rx, rbc_service_task) = - if dag_state.consensus_protocol.is_starfish_rbc() && !embedded_rbc_authority { + if dag_state.consensus_protocol.is_starfish_rbc() { let protocol_instance = node_parameters .starfish_rbc_protocol_instance .and_then(|bytes| RbcProtocolInstanceId::new(bytes).ok()) @@ -3218,48 +1946,26 @@ impl NetworkSyncer } else { ShadowWalSyncPolicyV1::EveryBatch }; - if embedded_rbc_authority { - start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - recovered_local_headers, - // The idle carrier pacemaker deliberately shares the - // resolved Starfish leader timeout. Application and - // embedded RBC phase carriers remain event-driven. - node_parameters.leader_timeout, - wal_sync_policy, - Arc::clone(&metrics), - rbc_dag_frontier_recovery_cursor, - !rbc_dag_clock_start_paused, - ) - } else { - let start = if rbc_dag_clock_start_paused { - start_starfish_rbc_dag_autonomous_clock_service_paused_with_metrics_v1 - } else { - start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1 - }; - start( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - recovered_local_headers, - node_parameters.leader_timeout, - wal_sync_policy, - Arc::clone(&metrics), - ) - } + start_starfish_rbc_dag_autonomous_clock_service_v1( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + recovered_local_headers, + // The idle carrier pacemaker deliberately shares the + // resolved Starfish leader timeout. Application and + // embedded RBC phase carriers remain event-driven. + node_parameters.leader_timeout, + wal_sync_policy, + ) } else { let wal_sync_policy = if node_parameters.starfish_rbc_dag_shadow_buffered_wal { ShadowWalSyncPolicyV1::OnShutdown } else { ShadowWalSyncPolicyV1::EveryBatch }; - start_starfish_rbc_dag_shadow_service_with_metrics_v1( + start_starfish_rbc_dag_shadow_service_v1( starfish_rbc_dag_shadow_wal, committee_context, dag_state.get_own_authority_index(), @@ -3267,14 +1973,10 @@ impl NetworkSyncer authorizer, recovered_local_headers, wal_sync_policy, - Arc::clone(&metrics), ) }; match started { Ok((service, events, task)) => (Some(service), Some(events), Some(task)), - Err(error) if embedded_rbc_authority => { - panic!("embedded RBC-DAG authority failed to start: {error}"); - } Err(error) => { invalidate_shadow_run(&metrics); tracing::error!("Disabling Starfish-RBC-DAG runtime: {error}"); @@ -3284,6 +1986,7 @@ impl NetworkSyncer } else { (None, None, None) }; + let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; let syncer = Syncer::new( core, NetworkSyncSignals { @@ -3300,11 +2003,10 @@ impl NetworkSyncer ); let initial_round = syncer.core().next_block_round(); let syncer = CoreThreadDispatcher::start(syncer); - if !embedded_rbc_authority { - // Embedded authority production is released by the ordered Ready - // bridge only after replayed frontier receipts are durable. - syncer.force_new_block(initial_round).await; - } + // Await the initial command while the async RBC actor remains + // schedulable. The command itself runs on the dedicated core thread, + // where synchronous local-INIT selection is safe. + syncer.force_new_block(initial_round).await; let (stop_sender, stop_receiver) = mpsc::channel(1); // Occupy the only available permit, so that all other // calls to send() will block. @@ -3348,17 +2050,6 @@ impl NetworkSyncer dag_state.attach_cordial_knowledge(cordial_knowledge_handle.clone()); let cordial_knowledge_task = handle.spawn(cordial_knowledge_actor.run()); - let (rbc_dag_clock_bridge_tx, rbc_dag_clock_bridge_state) = if node_parameters - .starfish_rbc_dag_autonomous_clock - && starfish_rbc_dag_shadow_service.is_some() - { - let (tx, rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - let rbc_dag_shutdown_started = Arc::new(AtomicBool::new(false)); - let inner = Arc::new(NetworkSyncerInner { block_ready_notify, dag_state: dag_state.clone(), @@ -3374,14 +2065,9 @@ impl NetworkSyncer cordial_knowledge: cordial_knowledge_handle, peer_senders: parking_lot::RwLock::new(AHashMap::new()), rbc_peer_senders: parking_lot::RwLock::new(AHashMap::new()), - rbc_dag_peer_mailboxes: parking_lot::RwLock::new(AHashMap::new()), leader_timeout: node_parameters.leader_timeout, soft_block_timeout: node_parameters.soft_block_timeout, - metrics: metrics.clone(), - rbc_dag_clock_bridge_tx: rbc_dag_clock_bridge_tx.clone(), - rbc_dag_shutdown_started: rbc_dag_shutdown_started.clone(), sailfish_handle: sf_handle_for_inner, - embedded_rbc_authority, starfish_rbc_service: starfish_rbc_service.clone(), starfish_rbc_dag_shadow_service: starfish_rbc_dag_shadow_service.clone(), start_time: std::time::Instant::now(), @@ -3483,6 +2169,17 @@ impl NetworkSyncer .syncer .add_transaction_data(vec![item], DataSource::StarfishRbcPayload) .await; + if let Some(ref shadow) = + event_inner.starfish_rbc_dag_shadow_service + { + if let Err(error) = shadow.application_data_available(block_ref) { + invalidate_shadow_run(&rbc_metrics); + tracing::warn!( + ?block_ref, + "Failed to record RBC-DAG application availability: {error}" + ); + } + } } RbcServiceEvent::Delivered(header) => { if let Some(ref shadow) = @@ -3530,129 +2227,43 @@ impl NetworkSyncer RbcCommitteeId::derive(&inner.committee) .expect("validated direct RBC committee must retain a stable identifier") }); - let (rbc_dag_core_control_tx, rbc_dag_core_control_task) = if embedded_rbc_authority { - let (control_tx, control_rx) = mpsc::channel(STARFISH_RBC_DAG_CORE_CONTROL_CAPACITY); - let worker_inner = inner.clone(); - let worker_metrics = metrics.clone(); - let worker_bridge_tx = rbc_dag_clock_bridge_tx - .as_ref() - .expect("embedded authority must supervise clock activation") - .clone(); - let panic_metrics = metrics.clone(); - let panic_bridge_tx = worker_bridge_tx.clone(); - let worker_shutdown_started = rbc_dag_shutdown_started.clone(); - let task = handle.spawn(async move { - if AssertUnwindSafe(run_rbc_dag_core_control_worker( - worker_inner, - worker_metrics, - worker_bridge_tx, - worker_shutdown_started, - control_rx, - )) - .catch_unwind() - .await - .is_err() - { - fail_rbc_dag_authority( - &panic_metrics, - Some(&panic_bridge_tx), - "RBC-DAG Core-control worker panicked", - ); - } - }); - (Some(control_tx), Some(task)) - } else { - (None, None) - }; - let (rbc_dag_assignment_tx, rbc_dag_assignment_task) = if embedded_rbc_authority { - // Capacity one plus the single in-flight dispatcher call preserves - // assignment order with a strict two-item bound, independently of - // remote payload verification in the authority FIFO. - let (assignment_tx, assignment_rx) = mpsc::channel(1); - let assignment_inner = inner.clone(); - let assignment_metrics = metrics.clone(); - let assignment_bridge_tx = rbc_dag_clock_bridge_tx - .as_ref() - .expect("embedded authority must supervise assignments") - .clone(); - let assignment_shutdown_started = rbc_dag_shutdown_started.clone(); - let panic_metrics = metrics.clone(); - let panic_bridge_tx = assignment_bridge_tx.clone(); - let task = handle.spawn(async move { - if AssertUnwindSafe(run_rbc_dag_application_assignment_worker( - assignment_inner, - assignment_metrics, - assignment_bridge_tx, - assignment_shutdown_started, - assignment_rx, - )) - .catch_unwind() - .await - .is_err() - { - fail_rbc_dag_authority( - &panic_metrics, - Some(&panic_bridge_tx), - "RBC-DAG assignment worker panicked", - ); - } - }); - (Some(assignment_tx), Some(task)) - } else { - (None, None) - }; let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); let shadow_metrics = metrics.clone(); - let rbc_dag_clock_bridge_tx = rbc_dag_clock_bridge_tx.clone(); - let rbc_dag_core_control_tx = rbc_dag_core_control_tx; - let rbc_dag_assignment_tx = rbc_dag_assignment_tx; - let rbc_dag_shutdown_started = rbc_dag_shutdown_started; handle.spawn(async move { - let mut router_guard = embedded_rbc_authority.then(|| { - RbcDagEventRouterGuardV1::new( - shadow_metrics.clone(), - rbc_dag_clock_bridge_tx - .as_ref() - .expect("embedded authority must supervise its event router") - .clone(), - ) - }); - // Keep expensive Reed-Solomon work off the carrier event - // router. Verification remains parallel, while the bounded - // Core-control worker awaits its handles in exact service - // order before applying a later frontier or activation. - let payload_verification_limit = Arc::new(Semaphore::new(4)); while let Some(event) = event_rx.recv().await { match event { ShadowServiceEventV1::Network { recipient, message } => { - // The per-peer keyed mailbox bounds both proactive - // history and exact-repair traffic. Distinct-key - // saturation is a transport failure: no proactive - // item is silently evicted to make room for a - // newer one. - let mailbox = event_inner - .rbc_dag_peer_mailboxes - .read() - .get(&recipient) - .cloned(); - if let Some(mailbox) = mailbox { - match mailbox.enqueue(message, &event_inner.committee) { - Ok(RbcDagOutboundEnqueueV1::Added) => shadow_metrics + let sender = event_inner.peer_senders.read().get(&recipient).cloned(); + if let Some(sender) = sender { + match sender.try_send(message) { + Ok(()) => shadow_metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["network", "sent"]) .inc(), - Ok(RbcDagOutboundEnqueueV1::Coalesced) => shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "coalesced"]) - .inc(), - Err(error) => fail_rbc_dag_outbound_transport( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - embedded_rbc_authority, - recipient, - &error, - ), + Err(mpsc::error::TrySendError::Full(_)) => { + // The shadow is observational. It must + // shed work instead of backpressuring + // the authoritative network path. + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "dropped_backpressure"]) + .inc(); + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["network", "disconnected"]) + .inc(); + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + } } } else { shadow_metrics @@ -3690,181 +2301,34 @@ impl NetworkSyncer ) { Ok(_) => {} Err(error) => { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - &format!( - "embedded carrier {carrier:?} delivered an invalid application header: {error}" - ), + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + tracing::error!( + ?carrier, + ?error, + "Embedded RBC delivered an invalid application header" ); } } } } - ShadowServiceEventV1::AuthorizedApplicationObserved { - carrier, - header, - payload, - authorization_basis, - } => { - if !embedded_rbc_authority { - tracing::debug!( - ?carrier, - ?authorization_basis, - "Ignoring standalone application event outside embedded authority mode" - ); - continue; - } - // Once authority has failed, do not even start a - // new payload verifier. Commands already accepted - // by the bounded FIFO are joined by its worker. - if rbc_dag_clock_bridge_failed(rbc_dag_clock_bridge_tx.as_ref()) { - continue; - } - let pinned = match PinnedRbcHeader::validate_with_committee_id( - header, - &event_inner.committee, - embedded_rbc_committee_id - .expect("embedded authority must cache its committee ID"), - ) { - Ok(pinned) => pinned, - Err(error) => { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - &format!( - "carrier {carrier:?} emitted an invalid authorized application header ({authorization_basis:?}): {error}" - ), - ); - continue; - } - }; - let canonical = pinned.header().clone(); - let block_ref = canonical.reference(); - let control_payload = if event_inner - .dag_state - .is_data_available(&block_ref) - { - let Some(block) = - event_inner.dag_state.get_storage_block(block_ref) - else { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - &format!( - "data-available application {block_ref} has no storage block" - ), - ); - continue; - }; - let transaction_data = block.transaction_data().cloned(); - if transaction_data.is_none() - && canonical.transactions_commitment() - != TransactionsCommitment::default() - { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - &format!( - "data-available application {block_ref} has no transaction payload" - ), - ); - continue; - } - // Empty committed transaction sets are - // data-available without a TransactionData - // allocation. The ordered availability callback - // remains required, but there is no payload to - // verify or cache. - RbcDagAuthorizedPayloadV1::AlreadyAvailable( - transaction_data.map(Arc::new), - ) - } else if let Some(transaction_data) = payload { - let payload_verification_limit = - payload_verification_limit.clone(); - let committee = event_inner.committee.clone(); - let mac_keys = event_inner.mac_keys.clone(); - let own_id = event_inner.dag_state.get_own_authority_index(); - let authentication_scheme = - event_inner.dag_state.block_authentication_scheme; - let verification_header = canonical.clone(); - let task = tokio::spawn(async move { - let permit = payload_verification_limit - .acquire_owned() - .await - .expect("RBC-DAG payload semaphore remains open"); - tokio::task::spawn_blocking(move || { - let _permit = permit; - let mut encoder = ReedSolomonEncoder::new(2, 4, 2) - .expect("RBC-DAG payload encoder should be created"); - verify_starfish_rbc_transaction_payload( - &verification_header, - transaction_data, - &committee, - own_id, - block_ref.authority, - &mut encoder, - authentication_scheme, - &mac_keys, - ) - }) - .await - }); - RbcDagAuthorizedPayloadV1::Verify(task) - } else { - RbcDagAuthorizedPayloadV1::None - }; - if let Some(ref control_tx) = rbc_dag_core_control_tx { - enqueue_rbc_dag_core_control( - control_tx, - RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier, - header: canonical, - authorization_basis, - payload: control_payload, - }, - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - ) - .await; - } - } - ShadowServiceEventV1::ApplicationAssigned(reference) => { - if let Some(ref assignment_tx) = rbc_dag_assignment_tx { - if rbc_dag_clock_bridge_failed(rbc_dag_clock_bridge_tx.as_ref()) { - continue; - } - // Assignment releases the one-outstanding - // producer gate and has no frontier/recovery - // dependency. Keep it off the payload-heavy - // FIFO; capacity one preserves exact order - // without spawning unbounded tasks. - if assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Assigned( - reference, - )) - .await - .is_err() - { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - "bounded RBC-DAG assignment channel closed unexpectedly", - ); - } - } - } ShadowServiceEventV1::FrontierCommitted(delta) => { - if let Some(ref control_tx) = rbc_dag_core_control_tx { - enqueue_rbc_dag_core_control( - control_tx, - RbcDagCoreControlCommandV1::Frontier(delta), - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - ) - .await; - } else { - shadow_metrics.starfish_rbc_dag_frontier_ignored(); + if embedded_rbc_authority { + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "committed"]) + .inc(); + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "carrier"]) + .inc_by(delta.carriers.len() as u64); + shadow_metrics + .starfish_rbc_dag_shadow_inputs_total + .with_label_values(&["frontier", "application"]) + .inc_by(delta.applications.len() as u64); + event_inner + .syncer + .apply_starfish_rbc_dag_frontier(delta) + .await; } } ShadowServiceEventV1::VertexProjected(reference) => { @@ -3952,43 +2416,13 @@ impl NetworkSyncer } } ShadowServiceEventV1::Ready { autonomous_clock } => { - if autonomous_clock && embedded_rbc_authority { - if let Some(ref control_tx) = rbc_dag_core_control_tx { - // FIFO placement makes this a barrier over - // every recovered header, payload, and - // committed frontier emitted before Ready. - enqueue_rbc_dag_core_control( - control_tx, - RbcDagCoreControlCommandV1::Ready, - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - ) - .await; - } + let verdict = if autonomous_clock { + &shadow_metrics.starfish_rbc_dag_shadow_clock_valid } else { - let verdict = - &shadow_metrics.starfish_rbc_dag_shadow_comparison_valid; - if verdict.get() != 0 { - verdict.set(1); - } - } - } - ShadowServiceEventV1::ClockActivated => { - if let Some(ref control_tx) = rbc_dag_core_control_tx { - enqueue_rbc_dag_core_control( - control_tx, - RbcDagCoreControlCommandV1::Activate, - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - ) - .await; - } else if shadow_metrics.starfish_rbc_dag_shadow_clock_valid.get() == 0 { - fail_rbc_dag_clock_bridge(rbc_dag_clock_bridge_tx.as_ref()); - } else { - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(1); - if let Some(ref bridge_tx) = rbc_dag_clock_bridge_tx { - bridge_tx.send_replace(RbcDagClockBridgeStateV1::Activated); - } + &shadow_metrics.starfish_rbc_dag_shadow_comparison_valid + }; + if verdict.get() != 0 { + verdict.set(1); } } ShadowServiceEventV1::ClockState { @@ -4032,17 +2466,10 @@ impl NetworkSyncer } ShadowServiceEventV1::Rejected { peer, error } => { if peer.is_none() { - if embedded_rbc_authority { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - &format!( - "RBC-DAG actor rejected authoritative runtime input: {error}" - ), - ); - } else { - invalidate_shadow_run(&shadow_metrics); - } + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } tracing::warn!( "Rejected RBC-DAG runtime input from {:?}: {}", @@ -4052,39 +2479,14 @@ impl NetworkSyncer } } } - if rbc_dag_shutdown_started.load(Ordering::Acquire) { - if let Some(ref assignment_tx) = rbc_dag_assignment_tx { - let _ = assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Drain) - .await; - } - } - if let Some(ref control_tx) = rbc_dag_core_control_tx { - if rbc_dag_shutdown_started.load(Ordering::Acquire) { - enqueue_rbc_dag_core_control( - control_tx, - RbcDagCoreControlCommandV1::Drain, - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - ) - .await; - } else { - fail_rbc_dag_authority( - &shadow_metrics, - rbc_dag_clock_bridge_tx.as_ref(), - "RBC-DAG actor event stream closed unexpectedly", - ); - } - } - if let Some(ref mut router_guard) = router_guard { - router_guard.record_clean_exit(); - } }) }); // Start bridge task that forwards reconstructed transaction data to core + let bridge_metrics = metrics.clone(); let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); + let bridge_metrics = bridge_metrics.clone(); handle.spawn(async move { while let Some(items) = decoded_rx.recv().await { // Reconstruction proves we now have the shard data for the @@ -4103,6 +2505,17 @@ impl NetworkSyncer .syncer .add_transaction_data(items, DataSource::ShardReconstructor) .await; + if let Some(ref shadow) = bridge_inner.starfish_rbc_dag_shadow_service { + for reference in shard_refs { + if let Err(error) = shadow.application_data_available(reference) { + invalidate_shadow_run(&bridge_metrics); + tracing::warn!( + ?reference, + "Failed to record reconstructed RBC-DAG availability: {error}" + ); + } + } + } } }) }); @@ -4371,170 +2784,15 @@ impl NetworkSyncer rbc_event_task, rbc_service_task, rbc_dag_shadow_event_task, - rbc_dag_core_control_task, - rbc_dag_assignment_task, rbc_dag_shadow_service_task, - rbc_dag_clock_bridge_state, cordial_knowledge_task, } } - pub(crate) async fn activate_starfish_rbc_dag_clock(&self) -> Result<(), String> { - let shadow = self - .inner - .starfish_rbc_dag_shadow_service - .clone() - .ok_or_else(|| "RBC-DAG clock service is not running".to_string())?; - let mut bridge_state = self - .rbc_dag_clock_bridge_state - .clone() - .ok_or_else(|| "RBC-DAG autonomous event bridge is not running".to_string())?; - match *bridge_state.borrow() { - RbcDagClockBridgeStateV1::Pending => {} - RbcDagClockBridgeStateV1::Failed => { - return Err( - "RBC-DAG startup recovery was invalid before clock activation".to_string(), - ); - } - RbcDagClockBridgeStateV1::Activated => return Ok(()), - } - - shadow - .activate_clock() - .await - .map_err(|error| format!("RBC-DAG clock activation failed: {error}"))?; - wait_for_rbc_dag_clock_bridge_activation(&mut bridge_state).await - } - - pub(crate) async fn shutdown(self) -> Option> { + pub(crate) async fn shutdown(self) -> Syncer { drop(self.stop); - // Stop new network/main ingress before asking the authoritative actor - // to close. The actor, ordered router, bounded workers, and Core must - // all remain alive until their already accepted work is drained. + // todo - wait for network shutdown as well self.main_task.await.ok(); - self.inner - .rbc_dag_shutdown_started - .store(true, Ordering::Release); - - let mut shadow_shutdown_timed_out = false; - if let Some(ref shadow) = self.inner.starfish_rbc_dag_shadow_service { - let shutdown_timeout = if self.inner.embedded_rbc_authority { - STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT - } else { - STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT - }; - match tokio::time::timeout(shutdown_timeout, shadow.shutdown()).await { - Ok(Ok(())) => {} - Ok(Err(error)) => { - tracing::warn!("RBC-DAG runtime did not acknowledge shutdown: {error}"); - if self.inner.embedded_rbc_authority { - fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - &format!("authoritative actor shutdown failed: {error}"), - ); - } - } - Err(_) => { - shadow_shutdown_timed_out = true; - tracing::warn!("Timed out stopping RBC-DAG runtime"); - if self.inner.embedded_rbc_authority { - fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - "authoritative actor shutdown timed out", - ); - } - } - } - } - - let mut rbc_dag_shadow_service_task = self.rbc_dag_shadow_service_task; - if let Some(mut actor_task) = rbc_dag_shadow_service_task.take() { - if shadow_shutdown_timed_out { - actor_task.abort(); - actor_task.await.ok(); - } else { - match tokio::time::timeout(STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT, &mut actor_task) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => { - if self.inner.embedded_rbc_authority { - fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - &format!( - "authoritative RBC-DAG actor supervisor failed during shutdown: {error}" - ), - ); - } else { - tracing::warn!( - "RBC-DAG actor supervisor failed during shutdown: {error}" - ); - } - } - Err(_) => { - fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - "RBC-DAG actor supervisor did not stop after shutdown", - ); - actor_task.abort(); - actor_task.await.ok(); - } - } - } - } - - if let Some(mut router_task) = self.rbc_dag_shadow_event_task { - match tokio::time::timeout(STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT, &mut router_task) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - &format!("RBC-DAG event router failed during shutdown: {error}"), - ), - Err(_) => { - fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - "RBC-DAG event router drain timed out", - ); - router_task.abort(); - router_task.await.ok(); - } - } - } - - for (name, task) in [ - ("assignment", self.rbc_dag_assignment_task), - ("Core-control", self.rbc_dag_core_control_task), - ] { - let Some(mut task) = task else { - continue; - }; - match tokio::time::timeout(STARFISH_RBC_DAG_CONTROL_DRAIN_TIMEOUT, &mut task).await { - Ok(Ok(())) => {} - Ok(Err(error)) => fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - &format!("RBC-DAG {name} worker failed during shutdown: {error}"), - ), - Err(_) => { - fail_rbc_dag_authority( - &self.inner.metrics, - self.inner.rbc_dag_clock_bridge_tx.as_ref(), - &format!("RBC-DAG {name} worker drain timed out"), - ); - task.abort(); - task.await.ok(); - } - } - } - // Close the shard reconstructor channel so the bridge task can exit // and release its Arc reference. self.inner.shard_tx.lock().take(); @@ -4572,6 +2830,11 @@ impl NetworkSyncer rbc_task.await.ok(); } let rbc_service_task = self.rbc_service_task; + if let Some(shadow_task) = self.rbc_dag_shadow_event_task { + shadow_task.abort(); + shadow_task.await.ok(); + } + let rbc_dag_shadow_service_task = self.rbc_dag_shadow_service_task; // Stop the cordial knowledge actor. self.cordial_knowledge_task.abort(); self.cordial_knowledge_task.await.ok(); @@ -4580,49 +2843,63 @@ impl NetworkSyncer // observe `stopped()`. Give them a short window to drop their `Arc`s // before insisting on `try_unwrap`. let mut inner_arc = self.inner; - let unwrap_deadline = Instant::now() + Duration::from_secs(2); + let mut attempts = 0usize; let inner = loop { match Arc::try_unwrap(inner_arc) { Ok(inner) => break inner, Err(arc) => { - if Instant::now() >= unwrap_deadline { - tracing::error!( - "Validator shutdown timed out waiting for auxiliary network workers" + attempts += 1; + if attempts >= 100 { + panic!( + "Shutdown failed - not all resources are freed \ + after main task is completed" ); - if let Some(task) = rbc_service_task { - task.abort(); - } - return None; } inner_arc = arc; - tokio::time::sleep(Duration::from_millis(1)).await; + tokio::task::yield_now().await; } } }; // `inner` is now exclusive, so no auxiliary task can enqueue after // this FIFO barrier. Awaiting it keeps the runtime available to the - // RBC actor while any earlier core action completes. The barrier is - // fallible because the core thread may concurrently panic while the - // remaining workers still need deterministic cleanup. - let _ = inner.syncer.flush_for_shutdown().await; - let syncer = match inner.syncer.stop() { - Ok(syncer) => Some(syncer), - Err(_) => { - tracing::error!("Core thread terminated before validator shutdown completed"); - None + // RBC actor while any earlier core action completes. + let _ = inner.syncer.missing_parent_references().await; + let mut shadow_shutdown_timed_out = false; + if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { + match tokio::time::timeout(STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT, shadow.shutdown()) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!("RBC-DAG runtime did not acknowledge shutdown: {error}") + } + Err(_) => { + shadow_shutdown_timed_out = true; + tracing::warn!( + "Timed out stopping RBC-DAG runtime; detaching it from validator shutdown" + ); + if let Some(task) = rbc_dag_shadow_service_task.as_ref() { + task.abort(); + } + } } - }; + } + let syncer = inner.syncer.stop(); if let Some(rbc_service_task) = rbc_service_task { rbc_service_task.abort(); rbc_service_task.await.ok(); } + if let Some(shadow_service_task) = rbc_dag_shadow_service_task { + match shadow_service_task.await { + Err(error) if !shadow_shutdown_timed_out => tracing::warn!( + "Non-authoritative RBC-DAG shadow supervisor failed during shutdown: {error}" + ), + _ => {} + } + } syncer } - pub(crate) fn is_finished(&self) -> bool { - self.main_task.is_finished() || self.inner.syncer.is_finished() - } - async fn run( mut network: Network, universal_committer: UniversalCommitter, @@ -4641,11 +2918,7 @@ impl NetworkSyncer None }; - // Embedded RBC-DAG frontiers are the sole commit authority. The - // legacy 10 ms commit poll is a no-op in that mode and otherwise - // needlessly submits roughly 100 core-thread commands per second. - let commit_timeout_task = (!inner.embedded_rbc_authority) - .then(|| handle.spawn(Self::commit_timeout_task(inner.clone()))); + let commit_timeout_task = handle.spawn(Self::commit_timeout_task(inner.clone())); let cleanup_task = handle.spawn(Self::cleanup_task( inner.clone(), bls_service.clone(), @@ -4698,8 +2971,12 @@ impl NetworkSyncer join_all( connections .into_values() - .chain([leader_timeout_task, cleanup_task, missing_parent_pull_task]) - .chain(commit_timeout_task) + .chain([ + leader_timeout_task, + commit_timeout_task, + cleanup_task, + missing_parent_pull_task, + ]) .chain(soft_block_timeout_task) .chain(cert_pull_task) .chain(round_gap_pull_task), @@ -4752,7 +3029,7 @@ impl NetworkSyncer .peer_senders .write() .insert(peer_id, connection.sender.clone()); - let rbc_outbound_task = inner.starfish_rbc_service.is_some().then(|| { + let rbc_outbound_task = inner.starfish_rbc_service.as_ref().map(|_| { let (rbc_sender, mut rbc_receiver) = mpsc::unbounded_channel(); inner.rbc_peer_senders.write().insert(peer_id, rbc_sender); let network_sender = connection.sender.clone(); @@ -4762,46 +3039,6 @@ impl NetworkSyncer } }) }); - let rbc_dag_outbound_task = inner.starfish_rbc_dag_shadow_service.is_some().then(|| { - let mailbox = RbcDagOutboundMailboxV1::new(); - inner - .rbc_dag_peer_mailboxes - .write() - .insert(peer_id, mailbox.clone()); - let proactive_sender = connection.rbc_dag_proactive_sender.clone(); - let priority_sender = connection.rbc_dag_priority_sender.clone(); - let outbound_failure = connection.outbound_failure.clone(); - let worker_metrics = shadow_metrics.clone(); - let worker_bridge_tx = inner.rbc_dag_clock_bridge_tx.clone(); - let authoritative = inner.embedded_rbc_authority; - Handle::current().spawn(async move { - if let Err(error) = run_rbc_dag_outbound_worker( - mailbox, - proactive_sender, - priority_sender, - outbound_failure, - ) - .await - { - if authoritative { - fail_rbc_dag_authority( - &worker_metrics, - worker_bridge_tx.as_ref(), - &format!( - "RBC-DAG outbound worker for authority {peer_id} failed: {error}" - ), - ); - } else { - invalidate_shadow_run(&worker_metrics); - tracing::error!( - peer = peer_id, - ?error, - "RBC-DAG observational outbound worker failed" - ); - } - } - }) - }); if let Some(ref rbc) = inner.starfish_rbc_service { if let Err(error) = rbc.peer_connected(peer_id) { tracing::warn!( @@ -4884,15 +3121,10 @@ impl NetworkSyncer } inner.peer_senders.write().remove(&peer_id); inner.rbc_peer_senders.write().remove(&peer_id); - inner.rbc_dag_peer_mailboxes.write().remove(&peer_id); if let Some(rbc_outbound_task) = rbc_outbound_task { rbc_outbound_task.abort(); rbc_outbound_task.await.ok(); } - if let Some(rbc_dag_outbound_task) = rbc_dag_outbound_task { - rbc_dag_outbound_task.abort(); - rbc_dag_outbound_task.await.ok(); - } inner.syncer.authority_connection(peer_id, false).await; handler.shutdown().await; block_fetcher.remove_authority(peer_id).await; @@ -5015,9 +3247,6 @@ impl NetworkSyncer inner: Arc>, metrics: Arc, ) -> Option<()> { - if inner.embedded_rbc_authority { - return None; - } const SCAN_INTERVAL: Duration = Duration::from_millis(500); const PEER_COUNT: usize = 2; @@ -5276,397 +3505,15 @@ impl SyncerSignals for NetworkSyncSignals { #[cfg(test)] mod tests { - use std::{collections::VecDeque, sync::Mutex}; - - use prometheus::Registry; use rand::{SeedableRng, rngs::StdRng}; - use tokio::sync::oneshot; use super::*; use crate::{ - crypto::{self, SignatureBytes}, + crypto::{self, SignatureBytes, TransactionsCommitment}, encoder::ShardEncoder, - network::{ - RbcDagShadowCarrier, RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, - }, - starfish_rbc_dag::{ - CarrierHeaderV1Args, ConsensusVertexReference, carrier_genesis_reference, - }, types::{BaseTransaction, BlockReference, Transaction, TransactionData}, }; - fn rbc_dag_outbound_test_carrier( - committee: &Committee, - creation_time_ns: TimestampNs, - ) -> (BlockReference, NetworkMessage) { - let candidate = CandidateCarrierV1::try_new( - CarrierHeaderV1Args { - author: 0, - carrier_round: 1, - own_prev: carrier_genesis_reference(0), - weak_parents: vec![carrier_genesis_reference(1), carrier_genesis_reference(2)], - transactions_commitment: TransactionsCommitment::default(), - application_header: None, - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns, - }, - committee, - ) - .unwrap(); - let reference = candidate.reference(); - ( - reference, - NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { - canonical_carrier: candidate.canonical_wire_bytes().unwrap(), - authentication_sidecar: vec![0xA5], - application_payload: None, - }), - ) - } - - fn rbc_dag_outbound_test_sync_request(round: RoundNumber) -> NetworkMessage { - NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { - author: 1, - round, - }) - } - - fn rbc_dag_outbound_test_sync_response(round: RoundNumber, marker: u8) -> NetworkMessage { - NetworkMessage::RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse { - author: 1, - round, - canonical_carrier: vec![marker], - authentication_sidecar: vec![marker.wrapping_add(1)], - }) - } - - #[derive(Default)] - struct TestRbcDagCoreControlStateV1 { - staged: Vec, - available: Vec<(BlockReference, bool)>, - materialized: Vec, - assignments: Vec, - frontier_results: VecDeque>, - activations: usize, - events: Vec<&'static str>, - } - - #[derive(Default)] - struct TestRbcDagCoreControlTargetV1 { - state: Mutex, - assignment_observed: Notify, - } - - impl RbcDagCoreControlTargetV1 for TestRbcDagCoreControlTargetV1 { - async fn stage_authorized_application( - &self, - header: RbcCanonicalHeader, - _authorization_basis: ShadowApplicationAuthorizationBasisV1, - ) -> Result<(), String> { - let mut state = self.state.lock().unwrap(); - state.staged.push(header.reference()); - state.events.push("header"); - Ok(()) - } - - fn restore_available_application( - &self, - application: BlockReference, - payload: Option>, - ) -> Result<(), String> { - let mut state = self.state.lock().unwrap(); - state.available.push((application, payload.is_some())); - state.events.push("available"); - Ok(()) - } - - async fn materialize_authorized_payload( - &self, - item: ReconstructedTransactionData, - ) -> Result<(), String> { - let mut state = self.state.lock().unwrap(); - state.materialized.push(item.block_reference); - state.events.push("payload"); - Ok(()) - } - - async fn apply_frontier(&self, _delta: CommittedFrontierDeltaV1) -> Result { - let mut state = self.state.lock().unwrap(); - state.events.push("frontier"); - state.frontier_results.pop_front().unwrap_or(Ok(true)) - } - - async fn activate_authority(&self) -> Result<(), String> { - let mut state = self.state.lock().unwrap(); - state.activations += 1; - state.events.push("activate"); - Ok(()) - } - - async fn apply_assignment(&self, reference: BlockReference) -> Result<(), String> { - let mut state = self.state.lock().unwrap(); - state.assignments.push(reference); - state.events.push("assignment"); - drop(state); - self.assignment_observed.notify_waiters(); - Ok(()) - } - } - - fn rbc_dag_worker_test_metrics() -> Arc { - let registry = Registry::new(); - let (metrics, _) = Metrics::new(®istry, None, Some("starfish-rbc"), None); - metrics.starfish_rbc_dag_shadow_clock_valid.set(-1); - metrics - } - - fn rbc_dag_worker_test_application() -> (RbcCanonicalHeader, ReconstructedTransactionData) { - let committee = Committee::new_test(vec![1; 4]); - let transactions = vec![BaseTransaction::Share(Transaction::new(vec![7; 64]))]; - let mut commitment_encoder = - ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); - let encoded = commitment_encoder.encode_transactions( - &transactions, - committee.info_length(), - committee.len() - committee.info_length(), - ); - let (commitment, _) = TransactionsCommitment::new_from_encoded_transactions(&encoded, 1); - let canonical = RbcCanonicalHeader::try_new( - 0, - 1, - vec![ - BlockReference::new_test(0, 0), - BlockReference::new_test(1, 0), - BlockReference::new_test(2, 0), - ], - Vec::new(), - 11, - commitment, - ) - .unwrap(); - let mut verifier = ReedSolomonEncoder::new(2, 4, 2).expect("encoder should be created"); - let item = verify_starfish_rbc_transaction_payload( - &canonical, - Arc::new(TransactionData::new(transactions)), - &committee, - 1, - 0, - &mut verifier, - BlockAuthenticationScheme::MacVector, - &[], - ) - .unwrap(); - (canonical, item) - } - - fn rbc_dag_worker_test_delta() -> CommittedFrontierDeltaV1 { - let carrier = BlockReference::new_test(0, 1); - CommittedFrontierDeltaV1 { - output_sequence: 1, - anchor: ConsensusVertexReference::new(carrier, 1), - frontier: Vec::new(), - carriers: Vec::new(), - applications: Vec::new(), - application_diagnostics: Vec::new(), - } - } - - #[test] - fn rbc_dag_outbound_mailbox_coalesces_exact_duplicates_and_rejects_conflicts() { - let committee = Committee::new_test(vec![1; 4]); - let mailbox = RbcDagOutboundMailboxV1::new(); - assert_eq!( - mailbox - .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) - .unwrap(), - RbcDagOutboundEnqueueV1::Added - ); - assert_eq!( - mailbox - .enqueue(rbc_dag_outbound_test_sync_response(7, 0xA1), &committee) - .unwrap(), - RbcDagOutboundEnqueueV1::Coalesced - ); - assert!(matches!( - mailbox.enqueue(rbc_dag_outbound_test_sync_response(7, 0xB1), &committee), - Err(RbcDagOutboundMailboxErrorV1::ConflictingDuplicate { - class: RbcDagOutboundClassV1::Priority, - key: RbcDagOutboundKeyV1::SyncResponse(1, 7), - }) - )); - let state = mailbox.inner.state.lock(); - assert_eq!(state.priority.entries.len(), 1); - assert_eq!(state.priority.order.len(), 1); - } - - #[test] - fn rbc_dag_outbound_mailbox_drains_priority_before_proactive() { - let committee = Committee::new_test(vec![1; 4]); - let mailbox = RbcDagOutboundMailboxV1::new(); - let (reference, proactive) = rbc_dag_outbound_test_carrier(&committee, 11); - mailbox.enqueue(proactive, &committee).unwrap(); - mailbox - .enqueue(rbc_dag_outbound_test_sync_request(9), &committee) - .unwrap(); - - let (class, first) = mailbox.try_pop().unwrap(); - assert_eq!(class, RbcDagOutboundClassV1::Priority); - assert!(matches!( - first, - NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { - author: 1, - round: 9, - }) - )); - let (class, second) = mailbox.try_pop().unwrap(); - assert_eq!(class, RbcDagOutboundClassV1::Proactive); - assert!(matches!( - rbc_dag_outbound_classification(&second, &committee).unwrap().1, - RbcDagOutboundKeyV1::Proactive(actual) if actual == reference - )); - } - - #[test] - fn rbc_dag_outbound_mailbox_never_evicts_a_unique_proactive_reference() { - let committee = Committee::new_test(vec![1; 4]); - let mailbox = RbcDagOutboundMailboxV1::new(); - let mut first_reference = None; - for marker in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY { - let (reference, message) = - rbc_dag_outbound_test_carrier(&committee, marker as TimestampNs); - first_reference.get_or_insert(reference); - assert_eq!( - mailbox.enqueue(message, &committee).unwrap(), - RbcDagOutboundEnqueueV1::Added - ); - } - let (_, overflow) = rbc_dag_outbound_test_carrier( - &committee, - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as TimestampNs + 1, - ); - assert!(matches!( - mailbox.enqueue(overflow, &committee), - Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { - class: RbcDagOutboundClassV1::Proactive, - capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - }) - )); - let state = mailbox.inner.state.lock(); - assert_eq!( - state.proactive.entries.len(), - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY - ); - assert!( - state - .proactive - .entries - .contains_key(&RbcDagOutboundKeyV1::Proactive(first_reference.unwrap())) - ); - } - - #[test] - fn rbc_dag_outbound_mailbox_bounds_distinct_priority_keys_and_bytes() { - let committee = Committee::new_test(vec![1; 4]); - let mailbox = RbcDagOutboundMailboxV1::new(); - for round in 1..=STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber { - mailbox - .enqueue(rbc_dag_outbound_test_sync_request(round), &committee) - .unwrap(); - } - assert!(matches!( - mailbox.enqueue( - rbc_dag_outbound_test_sync_request( - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY as RoundNumber + 1, - ), - &committee, - ), - Err(RbcDagOutboundMailboxErrorV1::KeyCapacity { - class: RbcDagOutboundClassV1::Priority, - capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - }) - )); - assert_eq!( - mailbox.inner.state.lock().priority.entries.len(), - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY - ); - - let first = rbc_dag_outbound_test_sync_request(1); - let framed_bytes = usize::try_from(bincode::serialized_size(&first).unwrap()) - .unwrap() - .checked_add(4) - .unwrap(); - let byte_bounded = RbcDagOutboundMailboxV1::from_state(RbcDagOutboundMailboxStateV1 { - priority: RbcDagOutboundLaneV1::new( - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - framed_bytes * 2 - 1, - framed_bytes, - ), - proactive: RbcDagOutboundLaneV1::new( - STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_BYTES, - STARFISH_RBC_DAG_OUTBOUND_PROACTIVE_ENTRY_BYTES, - ), - failure: None, - }); - byte_bounded.enqueue(first, &committee).unwrap(); - assert!(matches!( - byte_bounded.enqueue(rbc_dag_outbound_test_sync_request(2), &committee), - Err(RbcDagOutboundMailboxErrorV1::ByteCapacity { - class: RbcDagOutboundClassV1::Priority, - .. - }) - )); - let state = byte_bounded.inner.state.lock(); - assert_eq!(state.priority.entries.len(), 1); - assert_eq!(state.priority.bytes, framed_bytes); - } - - #[test] - fn rbc_dag_outbound_saturation_fails_authority_or_observation_explicitly() { - let error = RbcDagOutboundMailboxErrorV1::KeyCapacity { - class: RbcDagOutboundClassV1::Proactive, - capacity: STARFISH_RBC_DAG_OUTBOUND_KEY_CAPACITY, - }; - let authority_metrics = rbc_dag_worker_test_metrics(); - let (authority_bridge, authority_state) = - watch::channel(RbcDagClockBridgeStateV1::Activated); - fail_rbc_dag_outbound_transport( - &authority_metrics, - Some(&authority_bridge), - true, - 2, - &error, - ); - assert_eq!(*authority_state.borrow(), RbcDagClockBridgeStateV1::Failed); - assert_eq!( - authority_metrics.starfish_rbc_dag_shadow_clock_valid.get(), - 0 - ); - - let observation_metrics = rbc_dag_worker_test_metrics(); - let (observation_bridge, observation_state) = - watch::channel(RbcDagClockBridgeStateV1::Activated); - fail_rbc_dag_outbound_transport( - &observation_metrics, - Some(&observation_bridge), - false, - 2, - &error, - ); - assert_eq!( - *observation_state.borrow(), - RbcDagClockBridgeStateV1::Activated - ); - assert_eq!( - observation_metrics - .starfish_rbc_dag_shadow_clock_valid - .get(), - 0 - ); - } - #[tokio::test] async fn proposal_round_signal_notifies_waiters() { let block_ready_notify = Arc::new(Notify::new()); @@ -5681,475 +3528,6 @@ mod tests { wait.await; } - #[tokio::test] - async fn rbc_dag_clock_bridge_activation_ack_is_fail_closed() { - let (bridge_tx, mut bridge_state) = watch::channel(RbcDagClockBridgeStateV1::Pending); - fail_rbc_dag_clock_bridge(Some(&bridge_tx)); - - let error = wait_for_rbc_dag_clock_bridge_activation(&mut bridge_state) - .await - .unwrap_err(); - assert!(error.contains("startup recovery was invalid")); - } - - #[tokio::test] - async fn rbc_dag_clock_bridge_activation_ack_is_observable_without_a_lost_wakeup() { - let (bridge_tx, mut bridge_state) = watch::channel(RbcDagClockBridgeStateV1::Pending); - bridge_tx.send_replace(RbcDagClockBridgeStateV1::Activated); - - wait_for_rbc_dag_clock_bridge_activation(&mut bridge_state) - .await - .unwrap(); - - // A caller that already observed activation remains returned, while - // every later observer and authority command sees permanent failure. - fail_rbc_dag_clock_bridge(Some(&bridge_tx)); - assert_eq!( - *bridge_state.borrow_and_update(), - RbcDagClockBridgeStateV1::Failed - ); - } - - #[tokio::test] - async fn rbc_dag_assignment_lane_is_not_delayed_by_payload_verification() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, _bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let shutdown_started = Arc::new(AtomicBool::new(false)); - let (control_tx, control_rx) = mpsc::channel(1); - let (assignment_tx, assignment_rx) = mpsc::channel(1); - let control_task = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics.clone(), - bridge_tx.clone(), - shutdown_started.clone(), - control_rx, - )); - let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( - target.clone(), - metrics, - bridge_tx, - shutdown_started, - assignment_rx, - )); - let (release_payload, wait_for_payload) = oneshot::channel::<()>(); - let payload_task = tokio::spawn(async move { - wait_for_payload.await.unwrap(); - Ok(Err(eyre::eyre!("delayed untrusted payload"))) - }); - let (header, _) = rbc_dag_worker_test_application(); - control_tx - .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(1, 2), - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::Verify(payload_task), - }) - .await - .unwrap(); - - let assignment = BlockReference::new_test(0, 3); - let observed = target.assignment_observed.notified(); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Assigned(assignment)) - .await - .unwrap(); - tokio::time::timeout(Duration::from_millis(100), observed) - .await - .expect("assignment must bypass the blocked payload FIFO"); - assert_eq!(target.state.lock().unwrap().assignments, vec![assignment]); - - release_payload.send(()).unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Drain) - .await - .unwrap(); - control_task.await.unwrap(); - assignment_task.await.unwrap(); - } - - #[tokio::test] - async fn rbc_dag_router_fail_fast_blocks_assignment_while_payload_is_slow() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let shutdown_started = Arc::new(AtomicBool::new(false)); - let (control_tx, control_rx) = mpsc::channel(1); - let (assignment_tx, assignment_rx) = mpsc::channel(1); - let control_task = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics.clone(), - bridge_tx.clone(), - shutdown_started.clone(), - control_rx, - )); - let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( - target.clone(), - metrics.clone(), - bridge_tx.clone(), - shutdown_started, - assignment_rx, - )); - let (release_payload, wait_for_payload) = oneshot::channel::<()>(); - let (header, _) = rbc_dag_worker_test_application(); - control_tx - .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(1, 2), - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async move { - wait_for_payload.await.unwrap(); - Ok(Err(eyre::eyre!("slow untrusted payload"))) - })), - }) - .await - .unwrap(); - - // This models a router-known invalid header: failure is published - // synchronously instead of sitting behind the slow FIFO command. - fail_rbc_dag_authority( - &metrics, - Some(&bridge_tx), - "router rejected an invalid authoritative header", - ); - let assignment = BlockReference::new_test(0, 3); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Assigned(assignment)) - .await - .unwrap(); - tokio::task::yield_now().await; - assert_eq!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); - assert!(target.state.lock().unwrap().assignments.is_empty()); - - release_payload.send(()).unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Drain) - .await - .unwrap(); - control_task.await.unwrap(); - assignment_task.await.unwrap(); - } - - #[tokio::test] - async fn rbc_dag_worker_failure_blocks_later_assignment_at_apply_boundary() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - target - .state - .lock() - .unwrap() - .frontier_results - .push_back(Err("rejected frontier".to_owned())); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, mut bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let shutdown_started = Arc::new(AtomicBool::new(false)); - let (control_tx, control_rx) = mpsc::channel(2); - let (assignment_tx, assignment_rx) = mpsc::channel(1); - let control_task = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics.clone(), - bridge_tx.clone(), - shutdown_started.clone(), - control_rx, - )); - let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( - target.clone(), - metrics, - bridge_tx, - shutdown_started, - assignment_rx, - )); - - control_tx - .send(RbcDagCoreControlCommandV1::Frontier( - rbc_dag_worker_test_delta(), - )) - .await - .unwrap(); - tokio::time::timeout(Duration::from_millis(100), async { - while *bridge_rx.borrow_and_update() != RbcDagClockBridgeStateV1::Failed { - bridge_rx.changed().await.unwrap(); - } - }) - .await - .expect("frontier rejection must fail authority promptly"); - - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Assigned( - BlockReference::new_test(0, 3), - )) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Drain) - .await - .unwrap(); - control_task.await.unwrap(); - assignment_task.await.unwrap(); - - assert!(target.state.lock().unwrap().assignments.is_empty()); - } - - #[tokio::test] - async fn rbc_dag_bad_attached_payload_allows_later_valid_materialization() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let (control_tx, control_rx) = mpsc::channel(4); - let worker = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics, - bridge_tx, - Arc::new(AtomicBool::new(false)), - control_rx, - )); - let (header, valid_item) = rbc_dag_worker_test_application(); - let application = header.reference(); - control_tx - .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(1, 2), - header: header.clone(), - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async { - Ok(Err(eyre::eyre!("bad attached bytes"))) - })), - }) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(2, 3), - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async move { - Ok(Ok(valid_item)) - })), - }) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - worker.await.unwrap(); - - assert_ne!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); - let state = target.state.lock().unwrap(); - assert_eq!(state.staged, vec![application, application]); - assert_eq!(state.materialized, vec![application]); - } - - #[tokio::test] - async fn rbc_dag_exact_frontier_replay_has_no_applied_metric_or_effect() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - target - .state - .lock() - .unwrap() - .frontier_results - .push_back(Ok(false)); - let metrics = rbc_dag_worker_test_metrics(); - let applied_before = metrics - .starfish_rbc_dag_frontier_events_total - .with_label_values(&["applied"]) - .get(); - let (bridge_tx, _bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let (control_tx, control_rx) = mpsc::channel(2); - let worker = tokio::spawn(run_rbc_dag_core_control_worker( - target, - metrics.clone(), - bridge_tx, - Arc::new(AtomicBool::new(false)), - control_rx, - )); - control_tx - .send(RbcDagCoreControlCommandV1::Frontier( - rbc_dag_worker_test_delta(), - )) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - worker.await.unwrap(); - assert_eq!( - metrics - .starfish_rbc_dag_frontier_events_total - .with_label_values(&["applied"]) - .get(), - applied_before - ); - } - - #[tokio::test] - async fn rbc_dag_graceful_drain_applies_queued_frontier_before_exit() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let (control_tx, control_rx) = mpsc::channel(4); - let worker = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics, - bridge_tx, - Arc::new(AtomicBool::new(true)), - control_rx, - )); - let (release_payload, wait_for_payload) = oneshot::channel::<()>(); - let (header, item) = rbc_dag_worker_test_application(); - control_tx - .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(1, 2), - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::Verify(tokio::spawn(async move { - wait_for_payload.await.unwrap(); - Ok(Ok(item)) - })), - }) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Frontier( - rbc_dag_worker_test_delta(), - )) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - assert!(!worker.is_finished()); - release_payload.send(()).unwrap(); - worker.await.unwrap(); - - assert_ne!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); - assert_eq!( - target.state.lock().unwrap().events, - vec!["header", "payload", "frontier"] - ); - } - - #[tokio::test] - async fn rbc_dag_shutdown_skips_actor_callback_and_assignment_but_applies_frontier() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let shutdown_started = Arc::new(AtomicBool::new(true)); - let (control_tx, control_rx) = mpsc::channel(3); - let (assignment_tx, assignment_rx) = mpsc::channel(1); - let control_task = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics.clone(), - bridge_tx.clone(), - shutdown_started.clone(), - control_rx, - )); - let assignment_task = tokio::spawn(run_rbc_dag_application_assignment_worker( - target.clone(), - metrics, - bridge_tx, - shutdown_started, - assignment_rx, - )); - let (header, _) = rbc_dag_worker_test_application(); - control_tx - .send(RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(1, 2), - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::AlreadyAvailable(None), - }) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Frontier( - rbc_dag_worker_test_delta(), - )) - .await - .unwrap(); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Assigned( - BlockReference::new_test(0, 3), - )) - .await - .unwrap(); - control_tx - .send(RbcDagCoreControlCommandV1::Drain) - .await - .unwrap(); - assignment_tx - .send(RbcDagApplicationAssignmentCommandV1::Drain) - .await - .unwrap(); - control_task.await.unwrap(); - assignment_task.await.unwrap(); - - assert_ne!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Failed); - let state = target.state.lock().unwrap(); - assert!(state.available.is_empty()); - assert!(state.assignments.is_empty()); - assert_eq!(state.events, vec!["header", "frontier"]); - } - - #[tokio::test] - async fn rbc_dag_empty_available_application_precedes_ready_and_activation() { - let target = Arc::new(TestRbcDagCoreControlTargetV1::default()); - let metrics = rbc_dag_worker_test_metrics(); - let (bridge_tx, bridge_rx) = watch::channel(RbcDagClockBridgeStateV1::Pending); - let (control_tx, control_rx) = mpsc::channel(4); - let worker = tokio::spawn(run_rbc_dag_core_control_worker( - target.clone(), - metrics, - bridge_tx, - Arc::new(AtomicBool::new(false)), - control_rx, - )); - let header = RbcCanonicalHeader::try_new( - 0, - 1, - vec![ - BlockReference::new_test(0, 0), - BlockReference::new_test(1, 0), - BlockReference::new_test(2, 0), - ], - Vec::new(), - 11, - TransactionsCommitment::default(), - ) - .unwrap(); - let application = header.reference(); - for command in [ - RbcDagCoreControlCommandV1::AuthorizedApplicationObserved { - carrier: BlockReference::new_test(1, 2), - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - payload: RbcDagAuthorizedPayloadV1::AlreadyAvailable(None), - }, - RbcDagCoreControlCommandV1::Ready, - RbcDagCoreControlCommandV1::Activate, - RbcDagCoreControlCommandV1::Drain, - ] { - control_tx.send(command).await.unwrap(); - } - worker.await.unwrap(); - assert_eq!(*bridge_rx.borrow(), RbcDagClockBridgeStateV1::Activated); - let state = target.state.lock().unwrap(); - assert_eq!(state.available, vec![(application, false)]); - assert_eq!(state.events, vec!["header", "available", "activate"]); - } - #[test] fn starfish_rbc_initial_payload_is_commitment_checked() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 86856ae5..82805482 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -17,7 +17,7 @@ use tokio::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, }, runtime::Handle, - sync::{mpsc, watch}, + sync::{Mutex, mpsc}, time::Instant, }; @@ -32,16 +32,11 @@ use crate::{ stat::HistogramSender, types::{ AuthorityIndex, AuthoritySet, BlockReference, CertMessage, CertMessageKind, PartialSig, - ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, TransactionData, - VerifiedBlock, + ProvableShard, RoundNumber, SailfishNoVoteMsg, SailfishTimeoutMsg, VerifiedBlock, }, }; const PING_INTERVAL: Duration = Duration::from_secs(3); -pub(crate) const RBC_DAG_PRIORITY_CHANNEL_CAPACITY: usize = 64; -pub(crate) const RBC_DAG_PROACTIVE_CHANNEL_CAPACITY: usize = 64; -const NETWORK_SCHEDULED_LANE_CAPACITY: usize = 64; -const NETWORK_SCHEDULED_LANE_BYTE_CAPACITY: usize = 256 * 1024 * 1024; // Max buffer size controls the max amount of data (in bytes) to // be sent/received when sending batches of blocks. Based on the @@ -91,30 +86,13 @@ pub struct ShardPayload { /// Non-authoritative Starfish-RBC-DAG carrier used by the persisted shadow /// runtime. Both byte strings use the versioned canonical codecs from /// `starfish_rbc_dag`; the network envelope deliberately adds no second -/// identity or authentication scheme. The optional application payload is an -/// untrusted availability sidecar: receivers must verify it against the -/// transaction commitment in the carrier-authenticated application header. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// identity or authentication scheme. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct RbcDagShadowCarrier { pub canonical_carrier: Vec, pub authentication_sidecar: Vec, - pub application_payload: Option>, } -impl PartialEq for RbcDagShadowCarrier { - fn eq(&self, other: &Self) -> bool { - self.canonical_carrier == other.canonical_carrier - && self.authentication_sidecar == other.authentication_sidecar - && match (&self.application_payload, &other.application_payload) { - (Some(left), Some(right)) => left.transactions() == right.transactions(), - (None, None) => true, - (Some(_), None) | (None, Some(_)) => false, - } - } -} - -impl Eq for RbcDagShadowCarrier {} - /// Content-only response for a phase-evidenced shadow carrier. Recovery can /// satisfy READY/delivery, but it cannot grant optimistic admission or ECHO. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] @@ -143,15 +121,6 @@ pub struct RbcDagShadowCarrierSyncResponse { pub authentication_sidecar: Vec, } -/// Full transaction payload for one application header already authorized by -/// an authenticated or phase-evidenced embedded carrier. The payload itself -/// grants no authority and must be checked against the header commitment. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RbcDagApplicationPayloadResponse { - pub application: BlockReference, - pub transaction_data: Arc, -} - /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -273,12 +242,6 @@ pub enum NetworkMessage { /// Full canonical carrier and authentication sidecar for an exact /// carrier-clock slot. Receivers validate the duplicated slot identity. RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse), - /// Request transaction data for one exact application header already - /// authorized through the embedded carrier protocol. - RbcDagApplicationPayloadRequest(BlockReference), - /// Return commitment-checked transaction data for an authorized embedded - /// application header. The response is not an author proof. - RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse), } impl NetworkMessage { @@ -311,8 +274,6 @@ impl NetworkMessage { Self::RbcDagShadowCarrierResponse(_) => "rbc_dag_shadow_carrier_response", Self::RbcDagShadowCarrierSyncRequest(_) => "rbc_dag_shadow_carrier_sync_request", Self::RbcDagShadowCarrierSyncResponse(_) => "rbc_dag_shadow_carrier_sync_response", - Self::RbcDagApplicationPayloadRequest(_) => "rbc_dag_application_payload_request", - Self::RbcDagApplicationPayloadResponse(_) => "rbc_dag_application_payload_response", } } } @@ -325,31 +286,9 @@ pub struct Network { pub struct Connection { pub peer_id: usize, pub sender: mpsc::Sender, - /// Exact RBC-DAG repair remains distinct from ordinary/proactive traffic - /// until the socket scheduler makes its final priority decision. - pub(crate) rbc_dag_priority_sender: mpsc::Sender, - /// Proactive RBC-DAG carriers remain bounded independently of the - /// connection's legacy ordinary channel. - pub(crate) rbc_dag_proactive_sender: mpsc::Sender, - /// Terminal writer/scheduler failure for fail-closed authoritative users. - pub(crate) outbound_failure: watch::Receiver>, - /// Keep the connection-scoped failure signal open while this public - /// connection is alive. Cancelling the writer because the read half - /// disconnected or a newer socket replaced it is not a terminal writer - /// failure and must not look like one to authoritative users. - _outbound_failure_lifetime: watch::Sender>, pub receiver: mpsc::Receiver, } -impl Drop for Network { - fn drop(&mut self) { - // Dropping a Tokio JoinHandle detaches its task. Abort explicitly so - // a panic in the network-sync main loop cannot leave the listener - // alive and sharing SO_REUSEPORT with a later benchmark. - self.server_task.abort(); - } -} - impl Network { pub async fn load( parameters: &NodePublicConfig, @@ -378,12 +317,6 @@ impl Network { self.server_task.abort(); } - #[cfg(test)] - async fn abort_and_wait(mut self) -> Result<(), tokio::task::JoinError> { - self.server_task.abort(); - (&mut self.server_task).await - } - pub async fn from_socket_addresses( addresses: &[SocketAddr], our_id: usize, @@ -497,196 +430,6 @@ fn bind_addr(mut local_peer: SocketAddr) -> SocketAddr { const NETWORK_MESSAGE_CHANNEL_CAPACITY: usize = 1_000; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ScheduledNetworkClass { - Priority, - Ordinary, -} - -enum ScheduledNetworkPayload { - Message { - wire_bytes: Vec, - request_type: &'static str, - }, - Ping([u8; 12]), -} - -impl ScheduledNetworkPayload { - fn len(&self) -> usize { - match self { - Self::Message { wire_bytes, .. } => wire_bytes.len().saturating_add(4), - Self::Ping(bytes) => bytes.len(), - } - } -} - -#[cfg(test)] -fn scheduled_ping_value(payload: &ScheduledNetworkPayload) -> Option { - match payload { - ScheduledNetworkPayload::Ping(bytes) => Some(decode_ping(&bytes[4..])), - ScheduledNetworkPayload::Message { .. } => None, - } -} - -struct ScheduledNetworkWrite { - ready_at: Instant, - sequence: u64, - payload: ScheduledNetworkPayload, -} - -struct ScheduledNetworkLane { - writes: Vec, - bytes: usize, - count_capacity: usize, - byte_capacity: usize, -} - -impl ScheduledNetworkLane { - fn new(count_capacity: usize, byte_capacity: usize) -> Self { - Self { - writes: Vec::new(), - bytes: 0, - count_capacity, - byte_capacity, - } - } - - /// Reserve enough room for the largest legal frame before receiving the - /// next opaque `NetworkMessage`. This makes the post-serialization byte - /// bound strict without pulling an item out of a backpressured channel - /// that cannot yet be admitted. - fn can_receive_max_frame(&self) -> bool { - const MAX_FRAMED_BYTES: usize = MAX_BUFFER_SIZE as usize + 4; - self.writes.len() < self.count_capacity - && self.bytes <= self.byte_capacity.saturating_sub(MAX_FRAMED_BYTES) - } - - fn push(&mut self, write: ScheduledNetworkWrite) -> Result<(), ScheduledNetworkWrite> { - let bytes = write.payload.len(); - let Some(next_bytes) = self.bytes.checked_add(bytes) else { - return Err(write); - }; - if self.writes.len() >= self.count_capacity || next_bytes > self.byte_capacity { - return Err(write); - } - self.bytes = next_bytes; - self.writes.push(write); - Ok(()) - } - - fn ready_index(&self, now: Instant) -> Option { - self.writes - .iter() - .enumerate() - .filter(|(_, write)| write.ready_at <= now) - .min_by_key(|(_, write)| (write.ready_at, write.sequence)) - .map(|(index, _)| index) - } - - fn pop_ready(&mut self, now: Instant) -> Option { - let index = self.ready_index(now)?; - let write = self.writes.swap_remove(index); - self.bytes = self - .bytes - .checked_sub(write.payload.len()) - .expect("scheduled network byte accounting cannot underflow"); - Some(write) - } - - fn next_deadline(&self) -> Option { - self.writes.iter().map(|write| write.ready_at).min() - } - - fn is_empty(&self) -> bool { - self.writes.is_empty() - } -} - -struct ScheduledNetworkWrites { - priority: ScheduledNetworkLane, - ordinary: ScheduledNetworkLane, - next_sequence: u64, -} - -impl ScheduledNetworkWrites { - fn production() -> Self { - Self::new( - NETWORK_SCHEDULED_LANE_CAPACITY, - NETWORK_SCHEDULED_LANE_BYTE_CAPACITY, - ) - } - - fn new(lane_count_capacity: usize, lane_byte_capacity: usize) -> Self { - Self { - priority: ScheduledNetworkLane::new(lane_count_capacity, lane_byte_capacity), - ordinary: ScheduledNetworkLane::new(lane_count_capacity, lane_byte_capacity), - next_sequence: 0, - } - } - - fn lane(&self, class: ScheduledNetworkClass) -> &ScheduledNetworkLane { - match class { - ScheduledNetworkClass::Priority => &self.priority, - ScheduledNetworkClass::Ordinary => &self.ordinary, - } - } - - fn lane_mut(&mut self, class: ScheduledNetworkClass) -> &mut ScheduledNetworkLane { - match class { - ScheduledNetworkClass::Priority => &mut self.priority, - ScheduledNetworkClass::Ordinary => &mut self.ordinary, - } - } - - fn can_receive(&self, class: ScheduledNetworkClass) -> bool { - self.lane(class).can_receive_max_frame() - } - - fn push( - &mut self, - class: ScheduledNetworkClass, - ready_at: Instant, - payload: ScheduledNetworkPayload, - ) -> io::Result<()> { - let sequence = self.next_sequence; - self.next_sequence = self.next_sequence.saturating_add(1); - self.lane_mut(class) - .push(ScheduledNetworkWrite { - ready_at, - sequence, - payload, - }) - .map_err(|write| { - io::Error::new( - io::ErrorKind::OutOfMemory, - format!( - "scheduled {:?} network lane exhausted at {} bytes", - class, - write.payload.len() - ), - ) - }) - } - - fn pop_ready(&mut self, now: Instant) -> Option { - self.priority - .pop_ready(now) - .or_else(|| self.ordinary.pop_ready(now)) - } - - fn next_deadline(&self) -> Option { - match (self.priority.next_deadline(), self.ordinary.next_deadline()) { - (Some(left), Some(right)) => Some(left.min(right)), - (Some(deadline), None) | (None, Some(deadline)) => Some(deadline), - (None, None) => None, - } - } - - fn is_empty(&self) -> bool { - self.priority.is_empty() && self.ordinary.is_empty() - } -} - struct Worker { peer: SocketAddr, peer_id: usize, @@ -702,103 +445,11 @@ struct Worker { struct WorkerConnection { sender: mpsc::Sender, receiver: mpsc::Receiver, - rbc_dag_priority_receiver: mpsc::Receiver, - rbc_dag_proactive_receiver: mpsc::Receiver, - outbound_failure: watch::Sender>, metrics: Arc, peer_id: usize, compress_network: bool, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum OrdinaryAdmissionSource { - Proactive, - Legacy, -} - -impl OrdinaryAdmissionSource { - fn alternate(self) -> Self { - match self { - Self::Proactive => Self::Legacy, - Self::Legacy => Self::Proactive, - } - } - - fn try_receive( - self, - proactive_receiver: &mut mpsc::Receiver, - legacy_receiver: &mut mpsc::Receiver, - ) -> Result { - match self { - Self::Proactive => proactive_receiver.try_recv(), - Self::Legacy => legacy_receiver.try_recv(), - } - } -} - -fn admit_ordinary_fairly( - scheduled: &mut ScheduledNetworkWrites, - proactive_receiver: &mut mpsc::Receiver, - legacy_receiver: &mut mpsc::Receiver, - proactive_closed: &mut bool, - legacy_closed: &mut bool, - next_source: &mut OrdinaryAdmissionSource, - effective_latency: f64, - compress_network: bool, - metrics: &Metrics, -) -> io::Result { - if !scheduled.can_receive(ScheduledNetworkClass::Ordinary) { - return Ok(false); - } - let Some(message) = try_receive_ordinary_fairly( - proactive_receiver, - legacy_receiver, - proactive_closed, - legacy_closed, - next_source, - ) else { - return Ok(false); - }; - schedule_network_message( - scheduled, - ScheduledNetworkClass::Ordinary, - message, - effective_latency, - compress_network, - metrics, - )?; - Ok(true) -} - -fn try_receive_ordinary_fairly( - proactive_receiver: &mut mpsc::Receiver, - legacy_receiver: &mut mpsc::Receiver, - proactive_closed: &mut bool, - legacy_closed: &mut bool, - next_source: &mut OrdinaryAdmissionSource, -) -> Option { - for source in [*next_source, next_source.alternate()] { - if match source { - OrdinaryAdmissionSource::Proactive => *proactive_closed, - OrdinaryAdmissionSource::Legacy => *legacy_closed, - } { - continue; - } - match source.try_receive(proactive_receiver, legacy_receiver) { - Ok(message) => { - *next_source = source.alternate(); - return Some(message); - } - Err(mpsc::error::TryRecvError::Empty) => {} - Err(mpsc::error::TryRecvError::Disconnected) => match source { - OrdinaryAdmissionSource::Proactive => *proactive_closed = true, - OrdinaryAdmissionSource::Legacy => *legacy_closed = true, - }, - } - } - None -} - impl Worker { const ACTIVE_HANDSHAKE: u64 = 0xFEFE0000; const PASSIVE_HANDSHAKE: u64 = 0x0000AEAE; @@ -895,9 +546,6 @@ impl Worker { let WorkerConnection { sender, receiver, - rbc_dag_priority_receiver, - rbc_dag_proactive_receiver, - outbound_failure, metrics, peer_id, compress_network, @@ -914,26 +562,16 @@ impl Worker { correct committee?", ) .clone(); - let write_metrics = metrics.clone(); - let write_fut = async move { - let result = Self::handle_write_stream( - writer, - receiver, - rbc_dag_priority_receiver, - rbc_dag_proactive_receiver, - pong_receiver, - latency_sender, - write_metrics, - extra_connection_latency, - extra_connection_scaled, - compress_network, - ) - .await; - if let Err(error) = &result { - outbound_failure.send_replace(Some(error.to_string())); - } - result - } + let write_fut = Self::handle_write_stream( + writer, + receiver, + pong_receiver, + latency_sender, + metrics.clone(), + extra_connection_latency, + extra_connection_scaled, + compress_network, + ) .boxed(); let read_fut = Self::handle_read_stream(reader, sender, pong_sender, metrics, compress_network) @@ -944,10 +582,8 @@ impl Worker { } async fn handle_write_stream( - mut writer: OwnedWriteHalf, + writer: OwnedWriteHalf, mut receiver: mpsc::Receiver, - mut rbc_dag_priority_receiver: mpsc::Receiver, - mut rbc_dag_proactive_receiver: mpsc::Receiver, mut pong_receiver: mpsc::Receiver, latency_sender: HistogramSender, metrics: Arc, @@ -955,147 +591,209 @@ impl Worker { connection_scaled: bool, compress_network: bool, ) -> io::Result<()> { + // Use Arc and Mutex to share the writer safely across multiple tasks + let writer = Arc::new(Mutex::new(writer)); let start = Instant::now(); - let effective_latency = effective_latency(connection_latency, connection_scaled); - let mut scheduled = ScheduledNetworkWrites::production(); - let mut ordinary_closed = false; - let mut priority_closed = false; - let mut proactive_closed = false; - let mut pong_closed = false; - let mut ping_deadline = start + PING_INTERVAL; - let mut next_ordinary_source = OrdinaryAdmissionSource::Proactive; + let bytes_sent_total = metrics.bytes_sent_total.clone(); + let network_requests_sent_total = metrics.network_requests_sent_total.clone(); + let network_message_bytes_sent_total = metrics.network_message_bytes_sent_total.clone(); + + // Spawn the first task for handling pings + let writer_clone = Arc::clone(&writer); + let bytes_sent_total_clone = bytes_sent_total.clone(); + let ping_task = async move { + let mut ping_deadline = start + PING_INTERVAL; + loop { + tokio::time::sleep_until(ping_deadline).await; + ping_deadline += PING_INTERVAL; + + let ping_time = start.elapsed().as_micros() as i64; + assert!(ping_time > 0); // interval can't be 0 + + let ping = encode_ping(ping_time); + let latency = + generate_latency(effective_latency(connection_latency, connection_scaled)); + tokio::time::sleep(latency).await; + + if let Err(e) = writer_clone.lock().await.write_all(&ping).await { + tracing::error!("Failed to write ping: {e}"); + break; + } + bytes_sent_total_clone.inc_by(12); // ping is 12-byte sized + } + }; - loop { - // Keep socket liveness independent of application throughput. A - // continuously ready zero-latency data lane must not prevent the - // read half from handing us pings (and eventually blocking on its - // bounded pong channel), nor postpone our periodic ping forever. - let now = Instant::now(); - service_network_keepalive( - &mut scheduled, - &mut pong_receiver, - &mut pong_closed, - start, - now, - &mut ping_deadline, - effective_latency, - &latency_sender, - )?; - - // Always admit queued exact repair before ordinary traffic. The - // two scheduled lanes have independent 64-item/byte bounds, so a - // proactive burst cannot consume repair capacity. - while scheduled.can_receive(ScheduledNetworkClass::Priority) { - match rbc_dag_priority_receiver.try_recv() { - Ok(message) => schedule_network_message( - &mut scheduled, - ScheduledNetworkClass::Priority, - message, - effective_latency, - compress_network, - &metrics, - )?, - Err(mpsc::error::TryRecvError::Empty) => break, - Err(mpsc::error::TryRecvError::Disconnected) => { - priority_closed = true; - break; + // Spawn the second task for handling pong responses + let writer_clone = Arc::clone(&writer); + let bytes_sent_total_clone = bytes_sent_total.clone(); + let pong_task = async move { + while let Some(ping) = pong_receiver.recv().await { + if ping == 0 { + tracing::warn!("Invalid ping: {ping}"); + break; + } + if ping > 0 { + match ping.checked_neg() { + Some(pong) => { + let pong = encode_ping(pong); + let latency = generate_latency(effective_latency( + connection_latency, + connection_scaled, + )); + tokio::time::sleep(latency).await; + + if let Err(e) = writer_clone.lock().await.write_all(&pong).await { + tracing::error!("Failed to write pong: {e}"); + break; + } + bytes_sent_total_clone.inc_by(12); // pong is 12-byte sized + } + None => { + tracing::warn!("Invalid ping: {ping}"); + break; + } + } + } else { + match ping.checked_neg().and_then(|n| u64::try_from(n).ok()) { + Some(our_ping) => { + let time = start.elapsed().as_micros() as u64; + if let Some(delay) = time.checked_sub(our_ping) { + latency_sender.observe(Duration::from_micros(delay)); + } else { + tracing::warn!("Invalid ping: {ping}, greater than current time"); + break; + } + } + None => { + tracing::warn!("Invalid pong: {ping}"); + break; + } } } + // Yield to ensure responsiveness + tokio::task::yield_now().await; } - while admit_ordinary_fairly( - &mut scheduled, - &mut rbc_dag_proactive_receiver, - &mut receiver, - &mut proactive_closed, - &mut ordinary_closed, - &mut next_ordinary_source, - effective_latency, - compress_network, - &metrics, - )? {} - - if let Some(write) = scheduled.pop_ready(Instant::now()) { - write_scheduled_network_payload(&mut writer, write.payload, &metrics).await?; - continue; - } - if ordinary_closed && priority_closed && proactive_closed && scheduled.is_empty() { - return Ok(()); - } + }; - let next_write_deadline = scheduled.next_deadline(); - let next_deadline = - next_write_deadline.map_or(ping_deadline, |deadline| deadline.min(ping_deadline)); - tokio::select! { - biased; - message = rbc_dag_priority_receiver.recv(), - if !priority_closed - && scheduled.can_receive(ScheduledNetworkClass::Priority) => { - match message { - Some(message) => schedule_network_message( - &mut scheduled, - ScheduledNetworkClass::Priority, - message, - effective_latency, - compress_network, - &metrics, - )?, - None => priority_closed = true, + // Spawn the third task(s) for handling message sending. + // + // Important: keep encoding (serialize/compress) decoupled from timed + // socket writes. If we combine them, bursts of large writes can + // backpressure encoding and starve catch-up for late joiners under + // latency simulation. + if connection_latency == 0.0 { + let message_task = async move { + while let Some(message) = receiver.recv().await { + let request_type = message.request_type(); + let serialized = bincode::serialize(&message).expect("Serialization failed"); + let wire_bytes = if compress_network { + metrics + .bytes_uncompressed_sent_total + .inc_by(serialized.len() as u64); + lz4_flex::compress_prepend_size(&serialized) + } else { + serialized + }; + let framed_len = wire_bytes.len() as u64 + 4; + + match async { + let mut writer_guard = writer.lock().await; + writer_guard.write_u32(wire_bytes.len() as u32).await?; + + bytes_sent_total.inc_by(framed_len); + writer_guard.write_all(&wire_bytes).await } - } - pong = pong_receiver.recv(), - if !pong_closed - && scheduled.can_receive(ScheduledNetworkClass::Priority) => { - match pong { - Some(pong) => schedule_or_observe_pong( - &mut scheduled, - pong, - start, - Instant::now(), - effective_latency, - &latency_sender, - )?, - None => pong_closed = true, + .await + { + Ok(()) => { + network_requests_sent_total + .with_label_values(&[request_type]) + .inc(); + network_message_bytes_sent_total + .with_label_values(&[request_type]) + .inc_by(framed_len); + } + Err(e) => { + tracing::error!("Failed to write message: {e}"); + } } } - message = rbc_dag_proactive_receiver.recv(), - if !proactive_closed - && scheduled.can_receive(ScheduledNetworkClass::Ordinary) => { - match message { - Some(message) => { - schedule_network_message( - &mut scheduled, - ScheduledNetworkClass::Ordinary, - message, - effective_latency, - compress_network, - &metrics, - )?; - next_ordinary_source = OrdinaryAdmissionSource::Legacy; - } - None => proactive_closed = true, + }; + + // Wait for all tasks to complete. + let _ = tokio::join!(ping_task, pong_task, message_task); + return Ok(()); + } + + // When simulating latency, preserve the "many in-flight messages" + // behavior by sleeping inside per-message tasks, but keep concurrency + // bounded to avoid unbounded task buildup under heavy load. + const MAX_IN_FLIGHT: usize = NETWORK_MESSAGE_CHANNEL_CAPACITY * 64; + let message_task = async move { + let mut join_set = tokio::task::JoinSet::new(); + + while let Some(message) = receiver.recv().await { + while join_set.len() >= MAX_IN_FLIGHT { + if join_set.join_next().await.is_none() { + break; } } - message = receiver.recv(), - if !ordinary_closed - && scheduled.can_receive(ScheduledNetworkClass::Ordinary) => { - match message { - Some(message) => { - schedule_network_message( - &mut scheduled, - ScheduledNetworkClass::Ordinary, - message, - effective_latency, - compress_network, - &metrics, - )?; - next_ordinary_source = OrdinaryAdmissionSource::Proactive; + + let writer = writer.clone(); + let bytes_sent_total = bytes_sent_total.clone(); + let network_requests_sent_total = network_requests_sent_total.clone(); + let network_message_bytes_sent_total = network_message_bytes_sent_total.clone(); + let bytes_uncompressed_sent_total = metrics.bytes_uncompressed_sent_total.clone(); + let latency = + generate_latency(effective_latency(connection_latency, connection_scaled)); + let request_type = message.request_type(); + + join_set.spawn(async move { + let serialized = bincode::serialize(&message).expect("Serialization failed"); + let wire_bytes = if compress_network { + bytes_uncompressed_sent_total.inc_by(serialized.len() as u64); + lz4_flex::compress_prepend_size(&serialized) + } else { + serialized + }; + let framed_len = wire_bytes.len() as u64 + 4; + tokio::time::sleep(latency).await; + + match async { + let mut writer_guard = writer.lock().await; + writer_guard.write_u32(wire_bytes.len() as u32).await?; + + bytes_sent_total.inc_by(framed_len); + writer_guard.write_all(&wire_bytes).await + } + .await + { + Ok(()) => { + network_requests_sent_total + .with_label_values(&[request_type]) + .inc(); + network_message_bytes_sent_total + .with_label_values(&[request_type]) + .inc_by(framed_len); + } + Err(e) => { + tracing::error!("Failed to write message: {e}"); } - None => ordinary_closed = true, } + }); + } + + while let Some(result) = join_set.join_next().await { + if let Err(e) = result { + tracing::error!("An inner task failed: {e:?}"); } - _ = tokio::time::sleep_until(next_deadline) => {} } - } + }; + + // Wait for all tasks to complete. + let _ = tokio::join!(ping_task, pong_task, message_task); + + Ok(()) } async fn handle_read_stream( @@ -1180,27 +878,15 @@ impl Worker { mpsc::channel(NETWORK_MESSAGE_CHANNEL_CAPACITY); let (network_out_sender, network_out_receiver) = mpsc::channel(NETWORK_MESSAGE_CHANNEL_CAPACITY); - let (rbc_dag_priority_sender, rbc_dag_priority_receiver) = - mpsc::channel(RBC_DAG_PRIORITY_CHANNEL_CAPACITY); - let (rbc_dag_proactive_sender, rbc_dag_proactive_receiver) = - mpsc::channel(RBC_DAG_PROACTIVE_CHANNEL_CAPACITY); - let (outbound_failure, outbound_failure_receiver) = watch::channel(None); let connection = Connection { peer_id: self.peer_id, sender: network_out_sender, - rbc_dag_priority_sender, - rbc_dag_proactive_sender, - outbound_failure: outbound_failure_receiver, - _outbound_failure_lifetime: outbound_failure.clone(), receiver: network_in_receiver, }; self.connection_sender.send(connection).await.ok()?; Some(WorkerConnection { sender: network_in_sender, receiver: network_out_receiver, - rbc_dag_priority_receiver, - rbc_dag_proactive_receiver, - outbound_failure, metrics: self.metrics.clone(), peer_id: self.peer_id, compress_network: self.compress_network, @@ -1208,165 +894,6 @@ impl Worker { } } -fn schedule_network_message( - scheduled: &mut ScheduledNetworkWrites, - class: ScheduledNetworkClass, - message: NetworkMessage, - effective_latency: f64, - compress_network: bool, - metrics: &Metrics, -) -> io::Result<()> { - let request_type = message.request_type(); - let serialized = bincode::serialize(&message).map_err(|error| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("network message serialization failed: {error}"), - ) - })?; - let wire_bytes = if compress_network { - metrics - .bytes_uncompressed_sent_total - .inc_by(serialized.len() as u64); - lz4_flex::compress_prepend_size(&serialized) - } else { - serialized - }; - if wire_bytes.len() > MAX_BUFFER_SIZE as usize { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "serialized network message is {} bytes, maximum {}", - wire_bytes.len(), - MAX_BUFFER_SIZE - ), - )); - } - scheduled.push( - class, - Instant::now() + generate_latency(effective_latency), - ScheduledNetworkPayload::Message { - wire_bytes, - request_type, - }, - ) -} - -fn schedule_or_observe_pong( - scheduled: &mut ScheduledNetworkWrites, - ping: i64, - start: Instant, - now: Instant, - effective_latency: f64, - latency_sender: &HistogramSender, -) -> io::Result<()> { - if ping == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "received a zero ping", - )); - } - if ping > 0 { - let pong = ping - .checked_neg() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "ping cannot be negated"))?; - return scheduled.push( - ScheduledNetworkClass::Priority, - now + generate_latency(effective_latency), - ScheduledNetworkPayload::Ping(encode_ping(pong)), - ); - } - let our_ping = ping - .checked_neg() - .and_then(|value| u64::try_from(value).ok()) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid pong"))?; - let elapsed = now.saturating_duration_since(start).as_micros() as u64; - let delay = elapsed.checked_sub(our_ping).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "pong timestamp is greater than the local clock", - ) - })?; - latency_sender.observe(Duration::from_micros(delay)); - Ok(()) -} - -fn service_network_keepalive( - scheduled: &mut ScheduledNetworkWrites, - pong_receiver: &mut mpsc::Receiver, - pong_closed: &mut bool, - start: Instant, - now: Instant, - ping_deadline: &mut Instant, - effective_latency: f64, - latency_sender: &HistogramSender, -) -> io::Result<()> { - if !*pong_closed && scheduled.can_receive(ScheduledNetworkClass::Priority) { - match pong_receiver.try_recv() { - Ok(pong) => schedule_or_observe_pong( - scheduled, - pong, - start, - now, - effective_latency, - latency_sender, - )?, - Err(mpsc::error::TryRecvError::Empty) => {} - Err(mpsc::error::TryRecvError::Disconnected) => *pong_closed = true, - } - } - if now >= *ping_deadline { - if scheduled.can_receive(ScheduledNetworkClass::Priority) { - let ping_time = now.saturating_duration_since(start).as_micros() as i64; - if ping_time > 0 { - scheduled.push( - ScheduledNetworkClass::Priority, - now + generate_latency(effective_latency), - ScheduledNetworkPayload::Ping(encode_ping(ping_time)), - )?; - } - } - *ping_deadline = now + PING_INTERVAL; - } - Ok(()) -} - -async fn write_scheduled_network_payload( - writer: &mut OwnedWriteHalf, - payload: ScheduledNetworkPayload, - metrics: &Metrics, -) -> io::Result<()> { - match payload { - ScheduledNetworkPayload::Message { - wire_bytes, - request_type, - } => { - let wire_len = u32::try_from(wire_bytes.len()).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "network frame length exceeds u32", - ) - })?; - writer.write_u32(wire_len).await?; - writer.write_all(&wire_bytes).await?; - let framed_len = wire_bytes.len() as u64 + 4; - metrics.bytes_sent_total.inc_by(framed_len); - metrics - .network_requests_sent_total - .with_label_values(&[request_type]) - .inc(); - metrics - .network_message_bytes_sent_total - .with_label_values(&[request_type]) - .inc_by(framed_len); - } - ScheduledNetworkPayload::Ping(bytes) => { - writer.write_all(&bytes).await?; - metrics.bytes_sent_total.inc_by(bytes.len() as u64); - } - } - Ok(()) -} - /// Generates a latency table for a geodistributed network. /// `n` is the number of nodes. /// `seed` is a global seed used for deterministic generation. @@ -1497,310 +1024,12 @@ mod tests { use super::*; use crate::{ committee::Committee, - crypto::{AsBytes, MacTag, TransactionsCommitment, dummy_signer}, + crypto::{MacTag, TransactionsCommitment, dummy_signer}, starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, - types::{BaseTransaction, Transaction}, }; const NETWORK_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(6); - fn scheduled_test_message(marker: RoundNumber) -> ScheduledNetworkPayload { - let message = NetworkMessage::SubscribeBroadcastRequest(marker); - ScheduledNetworkPayload::Message { - wire_bytes: bincode::serialize(&message).unwrap(), - request_type: message.request_type(), - } - } - - fn scheduled_test_marker(write: ScheduledNetworkWrite) -> RoundNumber { - let ScheduledNetworkPayload::Message { wire_bytes, .. } = write.payload else { - panic!("expected a scheduled network message"); - }; - let NetworkMessage::SubscribeBroadcastRequest(marker) = - bincode::deserialize(&wire_bytes).unwrap() - else { - panic!("expected a scheduled subscription marker"); - }; - marker - } - - fn outbound_test_marker(message: NetworkMessage) -> RoundNumber { - let NetworkMessage::SubscribeBroadcastRequest(marker) = message else { - panic!("expected an outbound subscription marker"); - }; - marker - } - - #[test] - fn ordinary_admission_alternates_continuously_ready_sources() { - let (proactive_sender, mut proactive_receiver) = mpsc::channel(8); - let (legacy_sender, mut legacy_receiver) = mpsc::channel(8); - for marker in 10..14 { - proactive_sender - .try_send(NetworkMessage::SubscribeBroadcastRequest(marker)) - .unwrap(); - } - for marker in 20..24 { - legacy_sender - .try_send(NetworkMessage::SubscribeBroadcastRequest(marker)) - .unwrap(); - } - let mut proactive_closed = false; - let mut legacy_closed = false; - let mut next = OrdinaryAdmissionSource::Proactive; - let markers = (0..8) - .map(|_| { - outbound_test_marker( - try_receive_ordinary_fairly( - &mut proactive_receiver, - &mut legacy_receiver, - &mut proactive_closed, - &mut legacy_closed, - &mut next, - ) - .expect("both saturated sources should remain admissible"), - ) - }) - .collect::>(); - assert_eq!(markers, vec![10, 20, 11, 21, 12, 22, 13, 23]); - } - - #[test] - fn ordinary_admission_falls_through_without_losing_its_fair_turn() { - let (proactive_sender, mut proactive_receiver) = mpsc::channel(4); - let (legacy_sender, mut legacy_receiver) = mpsc::channel(4); - legacy_sender - .try_send(NetworkMessage::SubscribeBroadcastRequest(20)) - .unwrap(); - let mut proactive_closed = false; - let mut legacy_closed = false; - let mut next = OrdinaryAdmissionSource::Proactive; - assert_eq!( - outbound_test_marker( - try_receive_ordinary_fairly( - &mut proactive_receiver, - &mut legacy_receiver, - &mut proactive_closed, - &mut legacy_closed, - &mut next, - ) - .unwrap(), - ), - 20 - ); - proactive_sender - .try_send(NetworkMessage::SubscribeBroadcastRequest(10)) - .unwrap(); - assert_eq!( - outbound_test_marker( - try_receive_ordinary_fairly( - &mut proactive_receiver, - &mut legacy_receiver, - &mut proactive_closed, - &mut legacy_closed, - &mut next, - ) - .unwrap(), - ), - 10 - ); - } - - #[test] - fn keepalive_is_serviced_before_a_continuously_ready_data_lane() { - let (mut histogram, latency_sender) = crate::stat::histogram::(); - let (pong_sender, mut pong_receiver) = mpsc::channel(16); - pong_sender.try_send(7).unwrap(); - let start = Instant::now(); - let now = start + PING_INTERVAL; - let mut ping_deadline = now; - let mut pong_closed = false; - let mut scheduled = ScheduledNetworkWrites::new(64, NETWORK_SCHEDULED_LANE_BYTE_CAPACITY); - scheduled - .push( - ScheduledNetworkClass::Ordinary, - start, - scheduled_test_message(10), - ) - .unwrap(); - - service_network_keepalive( - &mut scheduled, - &mut pong_receiver, - &mut pong_closed, - start, - now, - &mut ping_deadline, - 0.0, - &latency_sender, - ) - .unwrap(); - - let first = scheduled.pop_ready(now).unwrap(); - assert_eq!(scheduled_ping_value(&first.payload), Some(-7)); - let second = scheduled.pop_ready(now).unwrap(); - assert_eq!( - scheduled_ping_value(&second.payload), - Some(PING_INTERVAL.as_micros() as i64) - ); - assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 10); - histogram.receive_all(); - assert_eq!(histogram.total_count(), 0); - assert_eq!(ping_deadline, now + PING_INTERVAL); - } - - #[test] - fn normal_writer_cancellation_does_not_close_failure_signal() { - let (writer_failure, worker_failure) = watch::channel(None::); - let connection_lifetime = writer_failure.clone(); - drop(writer_failure); - assert!(matches!(worker_failure.has_changed(), Ok(false))); - drop(connection_lifetime); - assert!(worker_failure.has_changed().is_err()); - } - - #[test] - fn scheduled_writer_preserves_ready_priority_under_zero_and_aws_latency() { - let now = Instant::now(); - for latency in [Duration::ZERO, Duration::from_millis(130)] { - let mut scheduled = ScheduledNetworkWrites::new(64, 1024 * 1024); - scheduled - .push( - ScheduledNetworkClass::Ordinary, - now + latency, - scheduled_test_message(10), - ) - .unwrap(); - scheduled - .push( - ScheduledNetworkClass::Priority, - now + latency, - scheduled_test_message(20), - ) - .unwrap(); - assert_eq!( - scheduled_test_marker(scheduled.pop_ready(now + latency).unwrap()), - 20 - ); - assert_eq!( - scheduled_test_marker(scheduled.pop_ready(now + latency).unwrap()), - 10 - ); - } - } - - #[test] - fn scheduled_writer_does_not_send_unready_priority_ahead_of_ready_ordinary() { - let now = Instant::now(); - let mut scheduled = ScheduledNetworkWrites::new(64, 1024 * 1024); - scheduled - .push( - ScheduledNetworkClass::Priority, - now + Duration::from_millis(200), - scheduled_test_message(20), - ) - .unwrap(); - scheduled - .push( - ScheduledNetworkClass::Ordinary, - now + Duration::from_millis(100), - scheduled_test_message(10), - ) - .unwrap(); - assert_eq!( - scheduled_test_marker( - scheduled - .pop_ready(now + Duration::from_millis(100)) - .unwrap() - ), - 10 - ); - assert!( - scheduled - .pop_ready(now + Duration::from_millis(199)) - .is_none() - ); - assert_eq!( - scheduled_test_marker( - scheduled - .pop_ready(now + Duration::from_millis(200)) - .unwrap() - ), - 20 - ); - } - - #[test] - fn scheduled_writer_bounds_each_lane_without_cross_lane_eviction() { - let now = Instant::now(); - let one = scheduled_test_message(1); - let framed_bytes = one.len(); - let mut scheduled = ScheduledNetworkWrites::new(2, framed_bytes * 2); - for marker in [1, 2] { - scheduled - .push( - ScheduledNetworkClass::Ordinary, - now, - scheduled_test_message(marker), - ) - .unwrap(); - } - assert!( - scheduled - .push( - ScheduledNetworkClass::Ordinary, - now, - scheduled_test_message(3), - ) - .is_err() - ); - // Saturating the proactive/ordinary lane cannot consume priority - // count or byte credit. - for marker in [10, 11] { - scheduled - .push( - ScheduledNetworkClass::Priority, - now, - scheduled_test_message(marker), - ) - .unwrap(); - } - assert_eq!(scheduled.priority.writes.len(), 2); - assert_eq!(scheduled.ordinary.writes.len(), 2); - assert_eq!(scheduled.priority.bytes, framed_bytes * 2); - assert_eq!(scheduled.ordinary.bytes, framed_bytes * 2); - assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 10); - assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 11); - assert_eq!(scheduled_test_marker(scheduled.pop_ready(now).unwrap()), 1); - } - - #[tokio::test] - async fn connection_priority_channel_is_bounded_and_failure_is_observable() { - let (priority_sender, mut priority_receiver) = - mpsc::channel(RBC_DAG_PRIORITY_CHANNEL_CAPACITY); - for marker in 0..RBC_DAG_PRIORITY_CHANNEL_CAPACITY as RoundNumber { - priority_sender - .try_send(NetworkMessage::SubscribeBroadcastRequest(marker)) - .unwrap(); - } - assert!(matches!( - priority_sender.try_send(NetworkMessage::SubscribeBroadcastRequest(999)), - Err(mpsc::error::TrySendError::Full(_)) - )); - assert!(priority_receiver.recv().await.is_some()); - priority_sender - .try_send(NetworkMessage::SubscribeBroadcastRequest(999)) - .unwrap(); - - let (failure_sender, mut failure_receiver) = watch::channel(None); - failure_sender.send_replace(Some("scheduler saturated".to_owned())); - failure_receiver.changed().await.unwrap(); - assert_eq!( - failure_receiver.borrow().as_deref(), - Some("scheduler saturated") - ); - } - async fn connected_pair( addresses: &[SocketAddr; 2], parameters: &NodeParameters, @@ -1865,8 +1094,19 @@ mod tests { // the server tasks makes listener release deterministic for this test; // dropping their worker senders must then cancel every scoped stream // future and its OwnedWriteHalf. - let (result_0, result_1) = - tokio::join!(network_0.abort_and_wait(), network_1.abort_and_wait()); + network_0.abort_server(); + network_1.abort_server(); + let Network { + connection_receiver: connection_receiver_0, + server_task: server_task_0, + } = network_0; + let Network { + connection_receiver: connection_receiver_1, + server_task: server_task_1, + } = network_1; + drop(connection_receiver_0); + drop(connection_receiver_1); + let (result_0, result_1) = tokio::join!(server_task_0, server_task_1); assert!(result_0.is_err_and(|error| error.is_cancelled())); assert!(result_1.is_err_and(|error| error.is_cancelled())); } @@ -1930,13 +1170,9 @@ mod tests { )); let request = NetworkMessage::RbcHeaderRequest(block_ref); let response = NetworkMessage::RbcHeaderResponse(header); - let application_payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( - Transaction::new(vec![0xAC; 8]), - )])); let shadow = NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { canonical_carrier: vec![0xA3, 0xA4], authentication_sidecar: vec![0xA5], - application_payload: Some(Arc::clone(&application_payload)), }); let shadow_request = NetworkMessage::RbcDagShadowCarrierRequest(block_ref); let shadow_response = @@ -1956,12 +1192,6 @@ mod tests { canonical_carrier: vec![0xA8, 0xA9], authentication_sidecar: vec![0xAA, 0xAB], }); - let payload_request = NetworkMessage::RbcDagApplicationPayloadRequest(block_ref); - let payload_response = - NetworkMessage::RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse { - application: block_ref, - transaction_data: application_payload, - }); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), @@ -1973,8 +1203,6 @@ mod tests { (shadow_response, 17, "rbc_dag_shadow_carrier_response"), (sync_request, 18, "rbc_dag_shadow_carrier_sync_request"), (sync_response, 19, "rbc_dag_shadow_carrier_sync_response"), - (payload_request, 20, "rbc_dag_application_payload_request"), - (payload_response, 21, "rbc_dag_application_payload_response"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); @@ -2016,83 +1244,6 @@ mod tests { )); } - #[test] - fn rbc_dag_application_payload_sidecars_roundtrip_exactly() { - let application = BlockReference::new_test(2, 17); - let transaction_data = Arc::new(TransactionData::new(vec![BaseTransaction::Share( - Transaction::new(vec![0xE1, 0xE2, 0xE3]), - )])); - let carrier = RbcDagShadowCarrier { - canonical_carrier: vec![0xC1, 0xC2], - authentication_sidecar: vec![0xD1], - application_payload: Some(Arc::clone(&transaction_data)), - }; - let encoded = bincode::serialize(&NetworkMessage::RbcDagShadowCarrier(carrier)).unwrap(); - let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); - let NetworkMessage::RbcDagShadowCarrier(decoded) = decoded else { - panic!("decoded a different network-message variant"); - }; - assert_eq!(decoded.canonical_carrier, vec![0xC1, 0xC2]); - assert_eq!(decoded.authentication_sidecar, vec![0xD1]); - let decoded_payload = decoded - .application_payload - .expect("application payload should survive the wire round trip"); - assert_eq!(decoded_payload.number_transactions(), 1); - let BaseTransaction::Share(transaction) = &decoded_payload.transactions()[0]; - assert_eq!(transaction.as_bytes(), &[0xE1, 0xE2, 0xE3]); - - let payloadless = RbcDagShadowCarrier { - canonical_carrier: vec![0xC3], - authentication_sidecar: vec![0xD2], - application_payload: None, - }; - let encoded = - bincode::serialize(&NetworkMessage::RbcDagShadowCarrier(payloadless)).unwrap(); - assert_eq!( - encoded, - vec![ - 15, 0, 0, 0, // frozen enum discriminant - 1, 0, 0, 0, 0, 0, 0, 0, 0xC3, // canonical carrier bytes - 1, 0, 0, 0, 0, 0, 0, 0, 0xD2, // authentication sidecar bytes - 0, // no application payload - ], - "payloadless carrier wire grammar changed", - ); - let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); - assert!(matches!( - decoded, - NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { - application_payload: None, - .. - }) - )); - - let request = NetworkMessage::RbcDagApplicationPayloadRequest(application); - assert_eq!(variant_index(&request), 20); - let encoded = bincode::serialize(&request).unwrap(); - let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); - assert!(matches!( - decoded, - NetworkMessage::RbcDagApplicationPayloadRequest(decoded) if decoded == application - )); - - let response = - NetworkMessage::RbcDagApplicationPayloadResponse(RbcDagApplicationPayloadResponse { - application, - transaction_data, - }); - assert_eq!(variant_index(&response), 21); - let encoded = bincode::serialize(&response).unwrap(); - let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); - let NetworkMessage::RbcDagApplicationPayloadResponse(decoded) = decoded else { - panic!("decoded a different network-message variant"); - }; - assert_eq!(decoded.application, application); - assert_eq!(decoded.transaction_data.number_transactions(), 1); - let BaseTransaction::Share(transaction) = &decoded.transaction_data.transactions()[0]; - assert_eq!(transaction.as_bytes(), &[0xE1, 0xE2, 0xE3]); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn scoped_connection_tasks_allow_immediate_same_port_rebind() { // Active sockets bind to listener_port * 10. Keep those derived ports diff --git a/crates/starfish-core/src/rocks_store.rs b/crates/starfish-core/src/rocks_store.rs index 90c1d834..6b8c040c 100644 --- a/crates/starfish-core/src/rocks_store.rs +++ b/crates/starfish-core/src/rocks_store.rs @@ -15,7 +15,7 @@ use crate::{ crypto::BlockDigest, dag_state::CommitData, data::Data, - store::{RbcDagFrontierReceipt, Store, validate_rbc_dag_frontier_commit_batch}, + store::Store, types::{ BlockHeader, BlockReference, ProvableShard, RoundNumber, TransactionData, VerifiedBlock, }, @@ -28,8 +28,6 @@ const CF_TX_DATA: &str = "tx_data"; const CF_SHARD_DATA: &str = "shard_data"; const CF_COMMITS: &str = "commits"; const CF_DUAL_DAG_CLEAN: &str = "sailfish_certified"; -const CF_RBC_DAG_FRONTIER_RECEIPT: &str = "rbc_dag_frontier_receipt"; -const LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY: &[u8] = b"latest"; pub struct RocksStore { db: Arc, @@ -146,7 +144,6 @@ impl RocksStore { ColumnFamilyDescriptor::new(CF_SHARD_DATA, Self::data_cf_options()), ColumnFamilyDescriptor::new(CF_COMMITS, Self::metadata_cf_options()), ColumnFamilyDescriptor::new(CF_DUAL_DAG_CLEAN, Self::metadata_cf_options()), - ColumnFamilyDescriptor::new(CF_RBC_DAG_FRONTIER_RECEIPT, Self::metadata_cf_options()), ]; let db = DB::open_cf_descriptors(&opts, path, cf_descriptors).map_err(io::Error::other)?; @@ -410,38 +407,6 @@ impl Store for RocksStore { .map_err(io::Error::other) } - fn store_commits_with_rbc_dag_receipt( - &self, - committed_sub_dags: Vec, - receipt: RbcDagFrontierReceipt, - ) -> io::Result<()> { - validate_rbc_dag_frontier_commit_batch(&committed_sub_dags, &receipt)?; - let receipt_bytes = receipt.to_bytes()?; - - let mut wb = rocksdb::WriteBatch::default(); - let cf_commits = self.cf(CF_COMMITS)?; - if committed_sub_dags.is_empty() { - let key = serialize(&receipt.carrier_anchor).map_err(io::Error::other)?; - wb.delete_cf(&cf_commits, key); - } else { - let commit_data = &committed_sub_dags[0]; - let key = serialize(&commit_data.leader).map_err(io::Error::other)?; - let value = serialize(commit_data).map_err(io::Error::other)?; - wb.put_cf(&cf_commits, key, value); - } - - let cf_receipt = self.cf(CF_RBC_DAG_FRONTIER_RECEIPT)?; - wb.put_cf( - &cf_receipt, - LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY, - receipt_bytes, - ); - - self.db - .write_opt(wb, &self.write_opts) - .map_err(io::Error::other) - } - fn get_commit(&self, reference: &BlockReference) -> io::Result> { let key = serialize(reference).map_err(io::Error::other)?; let cf_commits = self.cf(CF_COMMITS)?; @@ -458,22 +423,6 @@ impl Store for RocksStore { } } - fn read_latest_rbc_dag_frontier_receipt(&self) -> io::Result> { - let cf = self.cf(CF_RBC_DAG_FRONTIER_RECEIPT)?; - match self - .db - .get_cf_opt( - &cf, - LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY, - &Self::get_read_opts(), - ) - .map_err(io::Error::other)? - { - Some(bytes) => RbcDagFrontierReceipt::from_bytes(&bytes).map(Some), - None => Ok(None), - } - } - fn store_header_bytes(&self, reference: &BlockReference, bytes: &[u8]) -> io::Result<()> { let key = serialize(reference).map_err(io::Error::other)?; let cf = self.cf(CF_HEADERS)?; @@ -647,158 +596,3 @@ impl Store for RocksStore { Ok(refs) } } - -#[cfg(test)] -mod tests { - use tempfile::TempDir; - - use super::RocksStore; - use crate::{ - dag_state::CommitData, - store::{RbcDagFrontierReceipt, Store}, - types::{BlockReference, MAX_COMMITTEE_SIZE}, - }; - - fn commit(leader: BlockReference, committed_rounds: Vec) -> CommitData { - CommitData { - leader, - sub_dag: vec![BlockReference::new_test(1, leader.round)], - committed_rounds, - } - } - - fn assert_commit(store: &impl Store, expected: &CommitData) { - let actual = store - .get_commit(&expected.leader) - .expect("commit read should succeed") - .expect("commit should exist"); - assert_eq!(actual.leader, expected.leader); - assert_eq!(actual.sub_dag, expected.sub_dag); - assert_eq!(actual.committed_rounds, expected.committed_rounds); - } - - #[test] - fn rbc_dag_receipt_and_commits_are_atomic_and_latest_is_a_point_value() { - let temp_dir = TempDir::new().unwrap(); - let store = RocksStore::open(temp_dir.path()).unwrap(); - - let legacy_leader = BlockReference::new_test(2, 253); - let legacy_commit = commit(legacy_leader, vec![253; 4]); - store.store_commits(vec![legacy_commit.clone()]).unwrap(); - assert_commit(&store, &legacy_commit); - assert!( - store - .read_latest_rbc_dag_frontier_receipt() - .unwrap() - .is_none() - ); - - // A control-only frontier has no new application commits, but its - // durable cursor must still advance. - let first_anchor = BlockReference::new_test(7, 255); - let first_receipt = RbcDagFrontierReceipt { - carrier_anchor: first_anchor, - output_sequence: 255, - committed_rounds: vec![250, 251, 252, 253], - }; - let stale_first_commit = commit(first_anchor, first_receipt.committed_rounds.clone()); - store.store_commits(vec![stale_first_commit]).unwrap(); - assert!(store.get_commit(&first_anchor).unwrap().is_some()); - store - .store_commits_with_rbc_dag_receipt(Vec::new(), first_receipt.clone()) - .unwrap(); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(first_receipt) - ); - assert!(store.get_commit(&first_anchor).unwrap().is_none()); - - // The exact application commit is stored under the consensus carrier - // anchor so Core can reconstruct the compact receipt's application - // references after restart. - let second_anchor = BlockReference::new_test(7, 256); - let application_commit = commit(second_anchor, vec![255, 256, 255, 256]); - let second_receipt = RbcDagFrontierReceipt { - carrier_anchor: second_anchor, - output_sequence: 256, - committed_rounds: vec![255, 256, 255, 256], - }; - store - .store_commits_with_rbc_dag_receipt( - vec![application_commit.clone()], - second_receipt.clone(), - ) - .unwrap(); - assert_commit(&store, &application_commit); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(second_receipt.clone()) - ); - - // Mismatched/multiple application commits are rejected before either - // commit data or the latest receipt can change. - let mismatched = commit(BlockReference::new_test(2, 254), vec![255, 256, 255, 256]); - let mismatched_watermarks = commit(second_anchor, vec![1; 4]); - for invalid in [ - vec![mismatched], - vec![mismatched_watermarks], - vec![application_commit.clone(), application_commit.clone()], - ] { - let error = store - .store_commits_with_rbc_dag_receipt(invalid, second_receipt.clone()) - .unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(second_receipt.clone()) - ); - } - - // Reusing the exact anchor for a control-only marker atomically - // removes stale application CommitData, preserving absence semantics. - let control_receipt = RbcDagFrontierReceipt { - carrier_anchor: second_anchor, - output_sequence: 257, - committed_rounds: second_receipt.committed_rounds.clone(), - }; - store - .store_commits_with_rbc_dag_receipt(Vec::new(), control_receipt.clone()) - .unwrap(); - assert!(store.get_commit(&second_anchor).unwrap().is_none()); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(control_receipt.clone()) - ); - - // Receipt validation happens before the batch is submitted, so an - // invalid vector cannot partially write its application commit or - // replace the last valid cursor. - let rejected_leader = BlockReference::new_test(3, 257); - let rejected = commit(rejected_leader, vec![257; 4]); - for committed_rounds in [Vec::new(), vec![0; usize::from(MAX_COMMITTEE_SIZE) + 1]] { - let invalid = RbcDagFrontierReceipt { - carrier_anchor: BlockReference::new_test(7, 257), - output_sequence: 258, - committed_rounds, - }; - let error = store - .store_commits_with_rbc_dag_receipt(vec![rejected.clone()], invalid) - .unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - } - assert!(store.get_commit(&rejected_leader).unwrap().is_none()); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(control_receipt.clone()) - ); - - drop(store); - let reopened = RocksStore::open(temp_dir.path()).unwrap(); - assert_commit(&reopened, &legacy_commit); - assert!(reopened.get_commit(&second_anchor).unwrap().is_none()); - assert_eq!( - reopened.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(control_receipt) - ); - } -} diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index eddbb10a..a68f2fe7 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -8,12 +8,7 @@ //! decoding. Callers validate canonical bytes before journaling them; the //! reducer pins the exact byte strings and rejects any later alternative. -use std::{ - collections::{BTreeMap, BTreeSet}, - error::Error, - fmt, - sync::Arc, -}; +use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; use crate::types::{AuthorityIndex, BlockReference, RoundNumber}; @@ -112,8 +107,6 @@ impl DurableOutboundCarrierV1 { pub enum PhaseKindV1 { Echo, Ready, - Vote, - Ack, } /// Durable result of processing one entry in an enclosing phase batch. @@ -129,8 +122,6 @@ impl PhaseKindV1 { match statement { RbcPhaseStatementV1::Echo { .. } => Self::Echo, RbcPhaseStatementV1::Ready { .. } => Self::Ready, - RbcPhaseStatementV1::Vote { .. } => Self::Vote, - RbcPhaseStatementV1::Ack { .. } => Self::Ack, } } } @@ -183,20 +174,6 @@ pub enum JournalEventV1 { context: RbcDagContextV1, target: BlockReference, }, - LockVote { - context: RbcDagContextV1, - target: BlockReference, - }, - LockAck { - context: RbcDagContextV1, - target: BlockReference, - }, - /// The sender-honest or optimistic-ECHO fast-delivery predicate became - /// slot-global before the slower Q-READY certificate. - LockOptimisticDelivery { - context: RbcDagContextV1, - target: BlockReference, - }, LockDelivery { context: RbcDagContextV1, target: BlockReference, @@ -246,9 +223,6 @@ impl JournalEventV1 { | Self::LockEcho { context, .. } | Self::LockAdmission { context, .. } | Self::LockReady { context, .. } - | Self::LockVote { context, .. } - | Self::LockAck { context, .. } - | Self::LockOptimisticDelivery { context, .. } | Self::LockDelivery { context, .. } | Self::LockConsensusSlot { context, .. } | Self::LockLeaderChoice { context, .. } @@ -303,18 +277,11 @@ pub struct JournalSnapshotV1 { context: RbcDagContextV1, own_authority: AuthorityIndex, ingress: Vec, - /// Derived exact index over `ingress`. This is intentionally absent from - /// the durable format: replay reconstructs it while preserving the - /// ordered ingress records above. - authenticated_ingress_references: BTreeSet, retained_carriers: BTreeMap, own_carriers: BTreeMap, admission_locks: BTreeMap, echo_locks: BTreeMap, ready_locks: BTreeMap, - vote_locks: BTreeMap, - ack_locks: BTreeMap, - optimistic_delivery_locks: BTreeMap, delivery_locks: BTreeMap, consensus_slots: BTreeMap, leader_choices: BTreeMap, @@ -330,15 +297,11 @@ impl JournalSnapshotV1 { context, own_authority, ingress: Vec::new(), - authenticated_ingress_references: BTreeSet::new(), retained_carriers: BTreeMap::new(), own_carriers: BTreeMap::new(), admission_locks: BTreeMap::new(), echo_locks: BTreeMap::new(), ready_locks: BTreeMap::new(), - vote_locks: BTreeMap::new(), - ack_locks: BTreeMap::new(), - optimistic_delivery_locks: BTreeMap::new(), delivery_locks: BTreeMap::new(), consensus_slots: BTreeMap::new(), leader_choices: BTreeMap::new(), @@ -387,22 +350,10 @@ impl JournalSnapshotV1 { self.ready_locks.get(&slot).copied() } - pub fn vote_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.vote_locks.get(&slot).copied() - } - - pub fn ack_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.ack_locks.get(&slot).copied() - } - pub fn delivery_lock(&self, slot: RbcSlotKeyV1) -> Option { self.delivery_locks.get(&slot).copied() } - pub fn optimistic_delivery_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.optimistic_delivery_locks.get(&slot).copied() - } - pub fn consensus_slot(&self, round: RoundNumber) -> Option { self.consensus_slots.get(&round).copied() } @@ -463,41 +414,11 @@ impl JournalSnapshotV1 { JournalEventV1::LockAdmission { target, .. } => self.lock_admission(*target), JournalEventV1::LockEcho { target, .. } => self.lock_echo(*target), JournalEventV1::LockReady { target, .. } => self.lock_ready(*target), - JournalEventV1::LockVote { target, .. } => self.lock_vote(*target), - JournalEventV1::LockAck { target, .. } => self.lock_ack(*target), - JournalEventV1::LockOptimisticDelivery { target, .. } => { - self.ensure_retained(*target)?; - let slot = RbcSlotKeyV1::of(*target); - if self - .delivery_lock(slot) - .is_some_and(|certified| certified != *target) - { - return Err(JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::OptimisticDelivery, - slot, - }); - } - Self::lock_candidate( - &mut self.optimistic_delivery_locks, - *target, - LockKindV1::OptimisticDelivery, - ) - } JournalEventV1::LockDelivery { target, .. } => { self.ensure_retained(*target)?; - let slot = RbcSlotKeyV1::of(*target); - if self.ready_lock(slot) != Some(*target) { + if self.ready_lock(RbcSlotKeyV1::of(*target)) != Some(*target) { return Err(JournalErrorV1::DeliveryWithoutMatchingReady(*target)); } - if self - .optimistic_delivery_lock(slot) - .is_some_and(|delivered| delivered != *target) - { - return Err(JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Delivery, - slot, - }); - } Self::lock_candidate(&mut self.delivery_locks, *target, LockKindV1::Delivery) } JournalEventV1::LockConsensusSlot { @@ -562,7 +483,6 @@ impl JournalSnapshotV1 { canonical_carrier_wire, authentication_sidecar, }); - self.authenticated_ingress_references.insert(reference); Ok(()) } @@ -598,7 +518,11 @@ impl JournalSnapshotV1 { } fn lock_admission(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - if !self.authenticated_ingress_references.contains(&target) { + let authenticated_ingress = self + .ingress + .iter() + .any(|ingress| ingress.reference == target); + if !authenticated_ingress { return Err(JournalErrorV1::AdmissionWithoutAuthenticatedIngress(target)); } Self::lock_candidate(&mut self.admission_locks, target, LockKindV1::Admission) @@ -609,16 +533,6 @@ impl JournalSnapshotV1 { Self::lock_candidate(&mut self.ready_locks, target, LockKindV1::Ready) } - fn lock_vote(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - self.ensure_retained(target)?; - Self::lock_candidate(&mut self.vote_locks, target, LockKindV1::Vote) - } - - fn lock_ack(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - self.ensure_retained(target)?; - Self::lock_candidate(&mut self.ack_locks, target, LockKindV1::Ack) - } - fn ensure_retained(&self, reference: BlockReference) -> Result<(), JournalErrorV1> { if self.retained_carriers.contains_key(&reference) { Ok(()) @@ -826,17 +740,14 @@ impl JournalSnapshotV1 { return Err(JournalErrorV1::OutboundSidecarNotPersisted(reference)); } let candidate = outbound.candidate.clone(); - // The target author is excluded from ECHO/VOTE/ACK in the optimistic - // RBC. `FixOwnCarrier` is the durable authority lock for exposing its - // own exact carrier; only phase statements carried inside it require - // their corresponding local locks below. + if self.echo_lock(RbcSlotKeyV1::of(reference)) != Some(reference) { + return Err(JournalErrorV1::OutboundEchoNotLocked(reference)); + } for statement in candidate.header().phase_batch() { let target = statement.target(); let lock = match statement { RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), - RbcPhaseStatementV1::Vote { .. } => self.vote_lock(RbcSlotKeyV1::of(target)), - RbcPhaseStatementV1::Ack { .. } => self.ack_lock(RbcSlotKeyV1::of(target)), }; if lock != Some(target) { return Err(JournalErrorV1::OutboundPhaseNotLocked(*statement)); @@ -896,7 +807,6 @@ impl JournalSnapshotV1 { let outer_slot = RbcSlotKeyV1::of(outer); let authorized = self.own_carrier(outer.round) == Some(outer) || self.admission_lock(outer_slot) == Some(outer) - || self.optimistic_delivery_lock(outer_slot) == Some(outer) || self.delivery_lock(outer_slot) == Some(outer); if !authorized { return Err(JournalErrorV1::OuterCarrierNotAdmittedOrDelivered(outer)); @@ -928,8 +838,6 @@ impl JournalSnapshotV1 { let lock = match statement { RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), - RbcPhaseStatementV1::Vote { .. } => self.vote_lock(RbcSlotKeyV1::of(target)), - RbcPhaseStatementV1::Ack { .. } => self.ack_lock(RbcSlotKeyV1::of(target)), }; if lock != Some(target) { return Err(JournalErrorV1::OwnPhaseWithoutDurableLock(statement)); @@ -1013,9 +921,6 @@ pub enum LockKindV1 { Admission, Echo, Ready, - Vote, - Ack, - OptimisticDelivery, Delivery, } @@ -1065,6 +970,7 @@ pub enum JournalErrorV1 { OutboundSidecarNotPersisted(BlockReference), OutboundAuthenticationContextMismatch, OutboundAuthenticationCandidateMismatch(BlockReference), + OutboundEchoNotLocked(BlockReference), OutboundPhaseNotLocked(RbcPhaseStatementV1), OutboundConsensusSlotNotLocked { consensus_round: RoundNumber, @@ -1488,26 +1394,6 @@ mod tests { .unwrap(); } - fn assert_authenticated_ingress_index_matches_scan(snapshot: &JournalSnapshotV1) { - let scanned = snapshot - .authenticated_ingress() - .iter() - .map(AuthenticatedIngressRecordV1::reference) - .collect::>(); - assert_eq!(snapshot.authenticated_ingress_references, scanned); - for reference in scanned { - assert_eq!( - snapshot - .authenticated_ingress_references - .contains(&reference), - snapshot - .authenticated_ingress() - .iter() - .any(|record| record.reference() == reference) - ); - } - } - fn admit_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { authenticate_candidate(journal, candidate); journal @@ -1723,211 +1609,6 @@ mod tests { )); } - #[test] - fn authenticated_ingress_index_matches_scan_after_live_failures_and_duplicates() { - let mut journal = journal(); - let candidate = candidate(0, 1, 0x22, Vec::new(), None); - let reference = candidate.reference(); - - assert_eq!( - journal - .append(JournalEventV1::AuthenticatedIngress { - context: journal.context, - sequence: 0, - authenticated: authenticated_with_marker(&candidate, 1, 0xA2), - provenance: IngressProvenanceV1::Relayed { peer: 2 }, - }) - .unwrap_err(), - JournalErrorV1::AuthenticatedIngressContextMismatch - ); - assert_authenticated_ingress_index_matches_scan(journal.snapshot()); - assert!( - !journal - .snapshot() - .authenticated_ingress_references - .contains(&reference) - ); - - let authenticated = authenticated(&candidate, 1); - journal - .record_authenticated_ingress( - authenticated.clone(), - IngressProvenanceV1::Relayed { peer: 2 }, - ) - .unwrap(); - journal - .record_authenticated_ingress(authenticated, IngressProvenanceV1::Relayed { peer: 3 }) - .unwrap(); - - assert_eq!(journal.snapshot().authenticated_ingress().len(), 2); - assert_eq!(journal.snapshot().authenticated_ingress_references.len(), 1); - assert_authenticated_ingress_index_matches_scan(journal.snapshot()); - } - - #[test] - fn authenticated_ingress_index_reconstructs_a_long_ordered_sequence() { - const REPEATED_INGRESS: usize = 512; - const UNIQUE_INGRESS: usize = 16; - - let context = context(0xA1); - let mut durable_events = Vec::with_capacity(REPEATED_INGRESS + UNIQUE_INGRESS); - let mut ordered_references = Vec::with_capacity(REPEATED_INGRESS + UNIQUE_INGRESS); - let mut unique_references = Vec::with_capacity(UNIQUE_INGRESS); - let first_candidate = candidate(0, 1, 0x26, Vec::new(), None); - let first_reference = first_candidate.reference(); - let first_authenticated = authenticated(&first_candidate, 1); - unique_references.push(first_reference); - for index in 0..REPEATED_INGRESS { - ordered_references.push(first_reference); - durable_events.push(JournalEventV1::AuthenticatedIngress { - context, - sequence: durable_events.len() as u64, - authenticated: first_authenticated.clone(), - provenance: IngressProvenanceV1::Relayed { - peer: if index % 2 == 0 { 2 } else { 3 }, - }, - }); - } - for index in 1..UNIQUE_INGRESS { - let candidate = candidate( - 0, - RoundNumber::try_from(index + 1).unwrap(), - (index as u8).wrapping_mul(37), - Vec::new(), - None, - ); - let reference = candidate.reference(); - let authenticated = authenticated(&candidate, 1); - unique_references.push(reference); - ordered_references.push(reference); - durable_events.push(JournalEventV1::AuthenticatedIngress { - context, - sequence: durable_events.len() as u64, - authenticated, - provenance: IngressProvenanceV1::Relayed { peer: 2 }, - }); - } - - let ingress_events = durable_events.clone(); - let mut journal = - WriteAheadJournalV1::from_durable_events(context, 1, durable_events).unwrap(); - assert_eq!(journal.durable_events(), ingress_events); - assert_eq!( - journal - .snapshot() - .authenticated_ingress() - .iter() - .map(AuthenticatedIngressRecordV1::reference) - .collect::>(), - ordered_references - ); - assert_eq!( - journal.snapshot().authenticated_ingress_references.len(), - UNIQUE_INGRESS - ); - assert_authenticated_ingress_index_matches_scan(journal.snapshot()); - - for target in [ - unique_references[0], - unique_references[UNIQUE_INGRESS / 2], - unique_references[UNIQUE_INGRESS - 1], - ] { - journal - .append(JournalEventV1::LockAdmission { context, target }) - .unwrap(); - } - - let reopened = journal.restart().unwrap().restart().unwrap(); - assert_authenticated_ingress_index_matches_scan(reopened.snapshot()); - assert_eq!(reopened.snapshot(), journal.snapshot()); - } - - #[test] - fn authenticated_ingress_index_preserves_admission_conflicts_across_reopen() { - let mut journal = journal(); - let first_candidate = candidate(0, 7, 0x23, Vec::new(), None); - let conflicting_candidate = candidate(0, 7, 0x24, Vec::new(), None); - let absent_candidate = candidate(2, 9, 0x25, Vec::new(), None); - let first = first_candidate.reference(); - let conflicting = conflicting_candidate.reference(); - let absent = absent_candidate.reference(); - let slot = RbcSlotKeyV1::of(first); - - authenticate_candidate(&mut journal, &first_candidate); - authenticate_candidate(&mut journal, &conflicting_candidate); - assert_authenticated_ingress_index_matches_scan(journal.snapshot()); - assert!( - journal - .snapshot() - .authenticated_ingress_references - .contains(&first) - ); - assert!( - journal - .snapshot() - .authenticated_ingress_references - .contains(&conflicting) - ); - assert!( - !journal - .snapshot() - .authenticated_ingress_references - .contains(&absent) - ); - - journal - .append(JournalEventV1::LockAdmission { - context: journal.context, - target: first, - }) - .unwrap(); - assert_eq!( - journal - .append(JournalEventV1::LockAdmission { - context: journal.context, - target: conflicting, - }) - .unwrap_err(), - JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Admission, - slot, - } - ); - assert_eq!( - journal - .append(JournalEventV1::LockAdmission { - context: journal.context, - target: absent, - }) - .unwrap_err(), - JournalErrorV1::AdmissionWithoutAuthenticatedIngress(absent) - ); - - let mut reopened = journal.restart().unwrap(); - assert_authenticated_ingress_index_matches_scan(reopened.snapshot()); - assert_eq!( - reopened - .append(JournalEventV1::LockAdmission { - context: reopened.context, - target: conflicting, - }) - .unwrap_err(), - JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Admission, - slot, - } - ); - assert_eq!( - reopened - .append(JournalEventV1::LockAdmission { - context: reopened.context, - target: absent, - }) - .unwrap_err(), - JournalErrorV1::AdmissionWithoutAuthenticatedIngress(absent) - ); - } - #[test] fn crash_boundaries_preserve_each_slot_global_lock() { let own_candidate = candidate(1, 1, 0x11, Vec::new(), None); @@ -2144,7 +1825,7 @@ mod tests { } #[test] - fn optimistic_delivery_authorizes_unadmitted_outer_batch_and_survives_q_ready() { + fn only_admitted_conflict_processes_until_the_other_is_delivered() { let mut journal = journal(); let first_statement = RbcPhaseStatementV1::Echo { target: reference(0, 1, 0x63), @@ -2197,7 +1878,13 @@ mod tests { ); journal - .append(JournalEventV1::LockOptimisticDelivery { + .append(JournalEventV1::LockReady { + context: journal.context, + target: second, + }) + .unwrap(); + journal + .append(JournalEventV1::LockDelivery { context: journal.context, target: second, }) @@ -2219,33 +1906,6 @@ mod tests { }) .unwrap(); assert_eq!(journal.snapshot().phase_batch_cursor(second), 1); - assert_eq!( - journal - .restart() - .unwrap() - .snapshot() - .optimistic_delivery_lock(RbcSlotKeyV1::of(second)), - Some(second) - ); - - // The independent fallback certificate remains durable and may - // arrive after the fast-delivery latch without changing its value. - journal - .append(JournalEventV1::LockReady { - context: journal.context, - target: second, - }) - .unwrap(); - journal - .append(JournalEventV1::LockDelivery { - context: journal.context, - target: second, - }) - .unwrap(); - assert_eq!( - journal.snapshot().delivery_lock(RbcSlotKeyV1::of(second)), - Some(second) - ); } #[test] @@ -2427,192 +2087,6 @@ mod tests { ); } - #[test] - fn vote_and_ack_local_locks_are_phase_separate_slot_global_and_restart_safe() { - let mut journal = journal(); - let first_candidate = candidate(0, 1, 0xA4, Vec::new(), None); - let second_candidate = candidate(0, 1, 0xA5, Vec::new(), None); - let first = first_candidate.reference(); - let second = second_candidate.reference(); - let slot = RbcSlotKeyV1::of(first); - - assert_eq!( - journal - .append(JournalEventV1::LockVote { - context: journal.context, - target: first, - }) - .unwrap_err(), - JournalErrorV1::CarrierContentNotRetained(first) - ); - assert_eq!( - journal - .append(JournalEventV1::LockAck { - context: journal.context, - target: second, - }) - .unwrap_err(), - JournalErrorV1::CarrierContentNotRetained(second) - ); - - retain_candidate(&mut journal, &first_candidate); - retain_candidate(&mut journal, &second_candidate); - journal - .append(JournalEventV1::LockVote { - context: journal.context, - target: first, - }) - .unwrap(); - // ACK has its own local phase namespace; it need not match the local - // VOTE when independently sufficient evidence selects another value. - journal - .append(JournalEventV1::LockAck { - context: journal.context, - target: second, - }) - .unwrap(); - - assert!(matches!( - journal.append(JournalEventV1::LockVote { - context: journal.context, - target: second, - }), - Err(JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Vote, - slot: conflict_slot, - }) if conflict_slot == slot - )); - assert!(matches!( - journal.append(JournalEventV1::LockAck { - context: journal.context, - target: first, - }), - Err(JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Ack, - slot: conflict_slot, - }) if conflict_slot == slot - )); - - let restarted = journal.restart().unwrap().restart().unwrap(); - assert_eq!(restarted.snapshot().vote_lock(slot), Some(first)); - assert_eq!(restarted.snapshot().ack_lock(slot), Some(second)); - } - - #[test] - fn vote_and_ack_batches_durably_classify_counted_replay_and_equivocation() { - let mut journal = journal(); - let first = reference(0, 1, 0xA6); - let second = reference(0, 1, 0xA7); - let first_batch = [ - RbcPhaseStatementV1::Vote { target: first }, - RbcPhaseStatementV1::Ack { target: first }, - ]; - let replay_batch = first_batch; - let conflicting_batch = [ - RbcPhaseStatementV1::Vote { target: second }, - RbcPhaseStatementV1::Ack { target: second }, - ]; - let first_outer_candidate = candidate(2, 2, 0xA8, first_batch.to_vec(), None); - let replay_outer_candidate = candidate(2, 3, 0xA9, replay_batch.to_vec(), None); - let conflicting_outer_candidate = candidate(2, 4, 0xAA, conflicting_batch.to_vec(), None); - let first_outer = first_outer_candidate.reference(); - let replay_outer = replay_outer_candidate.reference(); - let conflicting_outer = conflicting_outer_candidate.reference(); - admit_candidate(&mut journal, &first_outer_candidate); - admit_candidate(&mut journal, &replay_outer_candidate); - admit_candidate(&mut journal, &conflicting_outer_candidate); - - // The journal binds each event to the exact candidate batch position; - // a valid statement from the wrong position cannot be applied. - assert_eq!( - journal - .append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: first_outer, - index: 0, - sender: 2, - statement: first_batch[1], - }) - .unwrap_err(), - JournalErrorV1::PhaseBatchEntryMismatch { - outer: first_outer, - index: 0, - } - ); - - let apply_batch = |outer, statements: [RbcPhaseStatementV1; 2], context| { - vec![ - JournalEventV1::ApplyPhaseStatement { - context, - outer, - index: 0, - sender: 2, - statement: statements[0], - }, - JournalEventV1::AdvancePhaseBatchCursor { - context, - outer, - index: 0, - }, - JournalEventV1::ApplyPhaseStatement { - context, - outer, - index: 1, - sender: 2, - statement: statements[1], - }, - JournalEventV1::AdvancePhaseBatchCursor { - context, - outer, - index: 1, - }, - ] - }; - for (outer, statements) in [ - (first_outer, first_batch), - (replay_outer, replay_batch), - (conflicting_outer, conflicting_batch), - ] { - let batch = journal - .validate_batch(apply_batch(outer, statements, journal.context)) - .unwrap(); - journal.commit_validated_batch(batch).unwrap(); - } - - for index in 0..2 { - assert_eq!( - journal - .snapshot() - .phase_statement_outcome(first_outer, index), - Some(AppliedPhaseOutcomeV1::Counted) - ); - assert_eq!( - journal - .snapshot() - .phase_statement_outcome(replay_outer, index), - Some(AppliedPhaseOutcomeV1::IgnoredReplay) - ); - assert_eq!( - journal - .snapshot() - .phase_statement_outcome(conflicting_outer, index), - Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) - ); - } - assert_eq!(journal.snapshot().phase_batch_cursor(first_outer), 2); - assert_eq!(journal.snapshot().phase_batch_cursor(replay_outer), 2); - assert_eq!(journal.snapshot().phase_batch_cursor(conflicting_outer), 2); - - let restarted = journal.restart().unwrap(); - assert_eq!(restarted.snapshot(), journal.snapshot()); - assert_eq!( - restarted - .snapshot() - .phase_statement_outcome(conflicting_outer, 1), - Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) - ); - } - #[test] fn replay_is_idempotent_and_foreign_namespace_fails_closed() { let mut journal = journal(); @@ -2753,90 +2227,6 @@ mod tests { journal.append(expose).unwrap(); } - #[test] - fn outbound_exposure_waits_for_vote_and_ack_locks_in_exact_batch_order() { - let mut journal = journal(); - let vote_target_candidate = candidate(0, 1, 0xB8, Vec::new(), None); - let ack_target_candidate = candidate(2, 1, 0xB9, Vec::new(), None); - let vote = RbcPhaseStatementV1::Vote { - target: vote_target_candidate.reference(), - }; - let ack = RbcPhaseStatementV1::Ack { - target: ack_target_candidate.reference(), - }; - let own_candidate = candidate(1, 2, 0xBA, vec![vote, ack], None); - let own = own_candidate.reference(); - retain_candidate(&mut journal, &vote_target_candidate); - retain_candidate(&mut journal, &ack_target_candidate); - prepare_outbound_for_exposure(&mut journal, &own_candidate); - let expose = JournalEventV1::ExposeOutbound { - context: journal.context, - reference: own, - }; - - assert_eq!( - journal.append(expose.clone()).unwrap_err(), - JournalErrorV1::OutboundPhaseNotLocked(vote) - ); - let apply_vote = JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: own, - index: 0, - sender: 1, - statement: vote, - }; - assert_eq!( - journal.append(apply_vote.clone()).unwrap_err(), - JournalErrorV1::OwnPhaseWithoutDurableLock(vote) - ); - journal - .append(JournalEventV1::LockVote { - context: journal.context, - target: vote.target(), - }) - .unwrap(); - journal.append(apply_vote).unwrap(); - journal - .append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer: own, - index: 0, - }) - .unwrap(); - assert_eq!( - journal.append(expose.clone()).unwrap_err(), - JournalErrorV1::OutboundPhaseNotLocked(ack) - ); - let apply_ack = JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: own, - index: 1, - sender: 1, - statement: ack, - }; - assert_eq!( - journal.append(apply_ack.clone()).unwrap_err(), - JournalErrorV1::OwnPhaseWithoutDurableLock(ack) - ); - journal - .append(JournalEventV1::LockAck { - context: journal.context, - target: ack.target(), - }) - .unwrap(); - journal.append(apply_ack).unwrap(); - journal - .append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer: own, - index: 1, - }) - .unwrap(); - journal.append(expose).unwrap(); - assert!(journal.snapshot().outbound(own).unwrap().exposed()); - assert_eq!(journal.snapshot().phase_batch_cursor(own), 2); - } - #[test] fn outbound_exposure_waits_for_matching_consensus_and_leader_locks() { let mut journal = journal(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs index 092d8c60..f734e527 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -63,18 +63,14 @@ const OPTION_NONE: u8 = 0; const OPTION_SOME: u8 = 1; const PHASE_ECHO: u8 = 0; const PHASE_READY: u8 = 1; -// Phase codes are append-only: persisted carrier headers and their digests -// depend on the original ECHO=0/READY=1 assignments. -const PHASE_VOTE: u8 = 2; -const PHASE_ACK: u8 = 3; const LEADER_NONE: u8 = 0; const LEADER_VOTE: u8 = 1; const LEADER_NO_VOTE: u8 = 2; const BLOCK_REFERENCE_SIZE: usize = 2 + 4 + 32; const PROTOCOL_INSTANCE_SIZE: usize = 32; const COMMITTEE_ID_SIZE: usize = 32; -const AUTHENTICATION_DOMAIN: &[u8; 19] = b"STARFISH_RBC_DAG_V2"; -const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_DAG_V2_COMMITTEE_ID"; +const AUTHENTICATION_DOMAIN: &[u8; 19] = b"STARFISH_RBC_DAG_V1"; +const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_DAG_V1_COMMITTEE_ID"; const CARRIER_AUTHENTICATION_KIND: u8 = 0; const AUTHENTICATION_BASE_SIZE: usize = 123; const AUTHENTICATION_MAC_SIZE: usize = AUTHENTICATION_BASE_SIZE + 2; @@ -88,17 +84,12 @@ std::thread_local! { pub enum RbcPhaseStatementV1 { Echo { target: BlockReference }, Ready { target: BlockReference }, - Vote { target: BlockReference }, - Ack { target: BlockReference }, } impl RbcPhaseStatementV1 { pub fn target(self) -> BlockReference { match self { - Self::Echo { target } - | Self::Ready { target } - | Self::Vote { target } - | Self::Ack { target } => target, + Self::Echo { target } | Self::Ready { target } => target, } } @@ -106,8 +97,6 @@ impl RbcPhaseStatementV1 { match self { Self::Echo { .. } => PHASE_ECHO, Self::Ready { .. } => PHASE_READY, - Self::Vote { .. } => PHASE_VOTE, - Self::Ack { .. } => PHASE_ACK, } } } @@ -1708,11 +1697,7 @@ fn validate_outer_header( } } - // Four protocol phases plus bounded spillover from prior carrier rounds. - // Six entries per authority keeps the honest four-phase fast path away - // from the canonical validation boundary without making the wire vector - // unbounded. - let phase_limit = usize::min(MAX_PHASE_STATEMENTS_V1, committee.len().saturating_mul(6)); + let phase_limit = usize::min(MAX_PHASE_STATEMENTS_V1, committee.len().saturating_mul(4)); if header.phase_batch.len() > phase_limit { return Err(RbcDagError::VectorTooLong { field: "phase statements", @@ -2072,8 +2057,6 @@ fn decode_header( phase_batch.push(match code { PHASE_ECHO => RbcPhaseStatementV1::Echo { target }, PHASE_READY => RbcPhaseStatementV1::Ready { target }, - PHASE_VOTE => RbcPhaseStatementV1::Vote { target }, - PHASE_ACK => RbcPhaseStatementV1::Ack { target }, other => return Err(RbcDagError::InvalidPhase(other)), }); } @@ -2706,63 +2689,6 @@ mod tests { )); } - #[test] - fn vote_and_ack_phase_codes_are_append_only_and_round_trip() { - let committee = Committee::new_test(vec![1; 4]); - let target = reference(2, 1, 0xA8); - assert_eq!(RbcPhaseStatementV1::Echo { target }.code(), 0); - assert_eq!(RbcPhaseStatementV1::Ready { target }.code(), 1); - assert_eq!(RbcPhaseStatementV1::Vote { target }.code(), 2); - assert_eq!(RbcPhaseStatementV1::Ack { target }.code(), 3); - - let mut args = args(&committee, 3, 2); - args.phase_batch = vec![ - RbcPhaseStatementV1::Vote { target }, - RbcPhaseStatementV1::Ack { target }, - ]; - let candidate = CandidateCarrierV1::try_new(args, &committee).unwrap(); - let content = candidate.canonical_content_bytes().unwrap(); - let wire = candidate.canonical_wire_bytes().unwrap(); - assert_eq!( - CandidateCarrierV1::decode_content(&content, &committee, Some(candidate.reference())) - .unwrap(), - candidate - ); - assert_eq!( - CandidateCarrierV1::decode_wire(&wire, &committee, Some(candidate.reference())) - .unwrap(), - candidate - ); - } - - #[test] - fn four_phase_batch_accepts_six_entries_per_authority_at_the_boundary() { - let committee = Committee::new_test(vec![1; 4]); - let mut args = args(&committee, 3, 8); - for round in 1..=6 { - let target = reference(0, round, round as u8); - args.phase_batch.extend([ - RbcPhaseStatementV1::Echo { target }, - RbcPhaseStatementV1::Vote { target }, - RbcPhaseStatementV1::Ack { target }, - RbcPhaseStatementV1::Ready { target }, - ]); - } - assert_eq!(args.phase_batch.len(), committee.len() * 6); - CandidateCarrierV1::try_new(args.clone(), &committee).unwrap(); - - args.phase_batch.push(RbcPhaseStatementV1::Echo { - target: reference(1, 1, 0xFF), - }); - assert!(matches!( - CandidateCarrierV1::try_new(args, &committee), - Err(RbcDagError::VectorTooLong { - field: "phase statements", - count: 25, - }) - )); - } - #[test] fn acknowledgment_compression_normalizes_and_rejects_duplicates() { let committee = Committee::new_test(vec![1; 4]); @@ -2866,21 +2792,21 @@ mod tests { let base = context.public_authentication_statement(candidate.reference()); assert_eq!( hex::encode(context.committee_id().as_bytes()), - "82804ac9a25b89ad8098c52ec0ec7cfe4250d23d672a627bdb5795eda8ec4b98" + "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62" ); assert_eq!( hex::encode(base), concat!( - "53544152464953485f5242435f4441475f56320002", + "53544152464953485f5242435f4441475f56310002", "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", - "82804ac9a25b89ad8098c52ec0ec7cfe4250d23d672a627bdb5795eda8ec4b98", + "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", "000300000002", "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f" ) ); assert_eq!( hex::encode(context.public_authentication_digest(candidate.reference())), - "79a89fb36b517a157591fa0d44755211c5dfccb604e10d0d52ee921a4e5dae48" + "26a0866c9c6938c9158495f23fd22b281db6371461b6aad109ad41329e5fa5c8" ); assert_eq!(&base[..19], AUTHENTICATION_DOMAIN); assert_eq!(base[19], CARRIER_AUTHENTICATION_KIND); @@ -2918,9 +2844,9 @@ mod tests { assert_eq!( hex::encode(mac_statement), concat!( - "53544152464953485f5242435f4441475f56320003", + "53544152464953485f5242435f4441475f56310003", "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", - "82804ac9a25b89ad8098c52ec0ec7cfe4250d23d672a627bdb5795eda8ec4b98", + "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", "000300000002", "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f", "0002" @@ -2930,7 +2856,7 @@ mod tests { let tag = key.compute_rbc_tag(&mac_statement); assert_eq!( hex::encode(tag.as_ref()), - "d0beb7274a49c4d1b26f0986a370c3455e04cf899ca2e0d4d1c8ec0707746f35" + "118209f3c2c3025918ae7f60fe5a04a94e639cd06d910d89c483035c014fff02" ); for (field, offset) in [ ("domain", 0), diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index b89b0642..344eccb1 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -29,13 +29,11 @@ use super::{ carrier_genesis_reference, }; -/// Executable-model runahead bounds. Admission stays close to the exact local -/// clock, while the wider authenticated-retention window lets a temporarily -/// descheduled validator catch up without forcing the healthy quorum to pace -/// itself to the slowest peer. These remain prototype resource parameters, -/// not production protocol constants. +/// Executable-model runahead bound. This is deliberately a model parameter, +/// not a production protocol constant; the runtime value remains a proof and +/// benchmarking decision. pub const EXECUTABLE_MODEL_ADMISSION_WINDOW_V1: RoundNumber = 2; -pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 64; +pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 4; const MODEL_LINEAGE_DERIVE_CONTEXT: &str = "starfish-rbc-dag-model-lineage-v1"; type ModelLineage = [u8; 32]; @@ -46,23 +44,6 @@ enum IngressAuthentication { CandidateOnly, } -/// Safety basis for the optimistic RBC fast-delivery latch. -/// -/// The first three predicates are authoritative delivery rules: the sender is -/// known honest, or the author-excluding optimistic ECHO threshold proves the -/// value unique and forces the VOTE/ACK/READY fallback to terminate on it. -/// `Delivered` records the slower `Q`-READY path when no fast predicate fired. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum DeliveryPromiseBasisV1 { - LocalFixed, - /// The target author has stake greater than the maximum Byzantine stake, - /// so a receiver-authenticated author value cannot equivocate. - HonestAuthor, - /// The author-excluding optimistic ECHO threshold `O = M + b` was met. - OptimisticEcho, - Delivered, -} - /// Observable effects of one deterministic reducer transition. #[derive(Clone, Debug, Eq, PartialEq)] pub enum ModelEffect { @@ -73,11 +54,6 @@ pub enum ModelEffect { }, /// The local Bracha instance delivered this exact carrier value. Delivered(BlockReference), - /// This exact value satisfied an authoritative optimistic-delivery - /// predicate. The slower `Delivered` effect still records `Q`-READY - /// certification independently. Projection additionally requires DA and - /// an exact closed carrier prefix. - DeliveryPromised(BlockReference), /// One exact author prefix advanced by one carrier. PrefixAdvanced { authority: AuthorityIndex, @@ -96,8 +72,7 @@ pub enum ModelEffect { pub enum ModelTraceEvent { /// The first authenticated value selected for a remote carrier slot. AdmissionLocked(BlockReference), - /// A locally generated ECHO, VOTE, ACK, or READY became slot-global and - /// immutable for its phase. + /// A locally generated ECHO or READY became slot-global and immutable. LocalPhaseLocked(RbcPhaseStatementV1), /// One exact entry of an enclosing carrier's authenticated phase log is /// about to be applied. Any lock enabled by that entry follows this event. @@ -129,13 +104,6 @@ pub enum ModelTraceEvent { consensus_round: RoundNumber, choice: LeaderChoiceV1, }, - /// One safety-preserving promise predicate became true. - /// This lock is emitted at most once for an exact carrier and immediately - /// precedes its `DeliveryPromised` effect. - DeliveryPromiseLocked { - target: BlockReference, - basis: DeliveryPromiseBasisV1, - }, /// Bracha delivery became slot-global and immutable. DeliveryLocked(BlockReference), /// Existing non-durable output retained in its exact reducer order. @@ -225,7 +193,6 @@ pub struct CarrierLifecycle { pub admitted: bool, pub phase_batch_processed: bool, pub delivered: bool, - pub certified_delivered: bool, pub data_available: bool, pub prefix_closed: bool, } @@ -305,7 +272,6 @@ struct CarrierRecord { admitted: bool, phase_batch_cursor: usize, delivered: bool, - certified_delivered: bool, data_available: bool, prefix_closed: bool, } @@ -318,7 +284,6 @@ impl CarrierRecord { admitted: false, phase_batch_cursor: 0, delivered: false, - certified_delivered: false, data_available, prefix_closed: false, } @@ -331,7 +296,6 @@ impl CarrierRecord { phase_batch_processed: self.phase_batch_cursor == self.carrier.header().phase_batch().len(), delivered: self.delivered, - certified_delivered: self.certified_delivered, data_available: self.data_available, prefix_closed: self.prefix_closed, } @@ -341,35 +305,25 @@ impl CarrierRecord { #[derive(Clone, Debug, Default, Eq, PartialEq)] struct RbcCandidateState { echoes: BTreeSet, - votes: BTreeSet, - acks: BTreeSet, readies: BTreeSet, + echo_quorum_observed: bool, + ready_validity_observed: bool, + ready_quorum_observed: bool, requested_holders: BTreeSet, } impl RbcCandidateState { fn holders(&self) -> BTreeSet { - self.echoes - .iter() - .chain(&self.votes) - .chain(&self.acks) - .chain(&self.readies) - .copied() - .collect() + self.echoes.union(&self.readies).copied().collect() } } #[derive(Clone, Debug, Default, Eq, PartialEq)] struct RbcSlotState { echoed: Option, - voted: Option, - acked: Option, readied: Option, delivered: Option, - certified_delivered: Option, echo_by_sender: BTreeMap, - vote_by_sender: BTreeMap, - ack_by_sender: BTreeMap, ready_by_sender: BTreeMap, candidates: BTreeMap, } @@ -377,37 +331,15 @@ struct RbcSlotState { #[derive(Clone, Copy)] enum RbcAction { NeedCarrier, - SendVote, - SendAck, SendReady, - CertifyDelivery, + Deliver, None, } -/// Exact integer thresholds for one target-author slot. -/// -/// `fault = floor((W - 1) / 3)` is the greatest integer Byzantine stake -/// strictly below one third. ECHO, VOTE, and ACK exclude the target author. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct RbcThresholds { - fault: Stake, - ready_validity: Stake, - ready_quorum: Stake, - optimistic: Option, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct OptimisticThresholds { - vote_from_echo: Stake, - converge: Stake, - promise_from_echo: Stake, -} - /// Pure local state machine used by the milestone-two simulations. #[derive(Clone)] pub struct RbcDagModel { committee: Arc, - total_committee_stake: Stake, committee_id: RbcDagCommitteeId, context: RbcDagContextV1, own_authority: AuthorityIndex, @@ -423,7 +355,6 @@ pub struct RbcDagModel { pending_phases: VecDeque, pending_phase_set: BTreeSet, pending_delivered_batch_replays: VecDeque, - delivery_promises: BTreeMap, prefix_tips: Vec, included_frontier: Vec>, included: BTreeSet, @@ -443,16 +374,6 @@ impl RbcDagModel { if context.committee_id() != committee_id { return Err(ModelError::ContextMismatch); } - let total_committee_stake = - committee.authorities().try_fold(0u64, |total, authority| { - total - .checked_add( - committee - .get_stake(authority) - .ok_or(ModelError::UnknownAuthority(authority))?, - ) - .ok_or(ModelError::InvalidCommittee) - })?; let prefix_tips = committee .authorities() .map(carrier_genesis_reference) @@ -460,7 +381,6 @@ impl RbcDagModel { Ok(Self { included_frontier: vec![None; committee.len()], committee, - total_committee_stake, committee_id, context, own_authority, @@ -476,7 +396,6 @@ impl RbcDagModel { pending_phases: VecDeque::new(), pending_phase_set: BTreeSet::new(), pending_delivered_batch_replays: VecDeque::new(), - delivery_promises: BTreeMap::new(), prefix_tips, included: BTreeSet::new(), }) @@ -651,7 +570,7 @@ impl RbcDagModel { let limit = self .committee .len() - .saturating_mul(6) + .saturating_mul(4) .min(MAX_PHASE_STATEMENTS_V1); self.pending_phases .iter() @@ -715,7 +634,7 @@ impl RbcDagModel { /// Fix a locally authored carrier. Its phase batch must be the exact /// bounded FIFO prefix returned by [`Self::pending_phase_batch`]; newly - /// generated phase statements are left for a later carrier. + /// generated ECHO is left for a later carrier. pub fn start_local_carrier( &mut self, authenticated: LocallyAuthenticatedCarrierV1, @@ -779,8 +698,8 @@ impl RbcDagModel { .collect(); let reference = carrier.reference(); // Preflight all fallible ingress checks before the proof-critical - // write order. The exact local carrier must be fixed before any local - // phase statement is authorized or any embedded statement is exposed. + // write order. The exact local carrier must be fixed before its local + // ECHO is authorized or any embedded phase statement is exposed. self.preflight_receive(&carrier)?; self.own_fixed.insert(round, reference); log.proof(ModelTraceEvent::LocalCarrierFixed(reference)); @@ -794,7 +713,6 @@ impl RbcDagModel { choice: vertex.leader_choice(), }); } - self.lock_delivery_promise(reference, DeliveryPromiseBasisV1::LocalFixed, log); for statement in &expected_phase_batch { self.pending_phase_set.remove(statement); } @@ -913,21 +831,10 @@ impl RbcDagModel { true } }; - if selected && self.target_author_is_honest(reference.authority) { - // This capability authenticates the exact target-author bytes - // to this receiver. If the author's stake exceeds F, the - // fault model itself rules out author equivocation. - self.lock_delivery_promise(reference, DeliveryPromiseBasisV1::HonestAuthor, log); - } if selected && self.in_admission_window(reference.round) { self.promote_authenticated(reference, log); } } - // A locally fixed promise is persisted before the carrier is inserted - // above. Activate every already-locked fast-delivery predicate only - // after exact content exists, and defer phase-batch replay until the - // current reducer input finishes. - self.activate_delivery_promise(reference, log); self.maybe_advance_fast_clock(log); self.drain_delivered_phase_batches(log); } @@ -997,33 +904,6 @@ impl RbcDagModel { .and_then(|slot| slot.delivered) } - /// Return the exact value that reached the fallback `Q`-READY - /// certificate, independently of an earlier optimistic delivery. - pub fn certified_delivered( - &self, - authority: AuthorityIndex, - round: RoundNumber, - ) -> Option { - self.rbc_slots - .get(&(round, authority)) - .and_then(|slot| slot.certified_delivered) - } - - pub fn certified_delivery_count(&self) -> usize { - self.rbc_slots - .values() - .filter(|slot| slot.certified_delivered.is_some()) - .count() - } - - /// Return the first durable fast-delivery basis for this exact carrier. - pub fn delivery_promise_basis( - &self, - reference: &BlockReference, - ) -> Option { - self.delivery_promises.get(reference).copied() - } - pub fn prefix_tip(&self, authority: AuthorityIndex) -> Option { let tip = *self.prefix_tips.get(authority as usize)?; (tip.round > 0).then_some(tip) @@ -1194,56 +1074,6 @@ impl RbcDagModel { }) } - fn voters_stake_excluding( - &self, - voters: &BTreeSet, - excluded: AuthorityIndex, - ) -> Stake { - voters.iter().fold(0, |stake, authority| { - if *authority == excluded { - stake - } else { - stake.saturating_add(self.authority_stake(*authority)) - } - }) - } - - fn rbc_thresholds(&self, target_author: AuthorityIndex) -> Option { - let author_stake = self.committee.get_stake(target_author)?; - let fault = self.total_committee_stake.checked_sub(1)? / 3; - let ready_validity = fault.checked_add(1)?; - let ready_quorum = self.total_committee_stake.checked_sub(fault)?; - let optimistic = if author_stake <= fault { - let non_author_stake = self.total_committee_stake.checked_sub(author_stake)?; - let residual_fault = fault.checked_sub(author_stake)?; - let vote_from_echo = non_author_stake.checked_div(2)?.checked_add(1)?; - // floor((U + b) / 2) without overflowing the intermediate sum. - let converge = (non_author_stake / 2) - .checked_add(residual_fault / 2)? - .checked_add((non_author_stake % 2 + residual_fault % 2) / 2)? - .checked_add(1)?; - let promise_from_echo = vote_from_echo.checked_add(residual_fault)?; - Some(OptimisticThresholds { - vote_from_echo, - converge, - promise_from_echo, - }) - } else { - None - }; - Some(RbcThresholds { - fault, - ready_validity, - ready_quorum, - optimistic, - }) - } - - fn target_author_is_honest(&self, target_author: AuthorityIndex) -> bool { - self.rbc_thresholds(target_author) - .is_some_and(|thresholds| thresholds.optimistic.is_none()) - } - fn rbc_slot_mut(&mut self, reference: BlockReference) -> &mut RbcSlotState { self.rbc_slots .entry((reference.round, reference.authority)) @@ -1252,15 +1082,6 @@ impl RbcDagModel { fn authorize_local_echo(&mut self, reference: BlockReference, log: &mut TransitionLog) { let own = self.own_authority; - if own == reference.authority { - // The target author is excluded from ECHO/VOTE/ACK. A locally - // fixed high-stake author instead seeds READY: its stake is at - // least F+1, so every correct receiver can safely amplify it. - if self.target_author_is_honest(reference.authority) { - self.authorize_local_ready(reference, log); - } - return; - } let slot = self.rbc_slot_mut(reference); if slot.echoed.is_some() { return; @@ -1278,25 +1099,6 @@ impl RbcDagModel { self.drive_rbc(reference, log); } - fn authorize_local_ready(&mut self, reference: BlockReference, log: &mut TransitionLog) { - let own = self.own_authority; - let slot = self.rbc_slot_mut(reference); - if slot.readied.is_some() { - return; - } - slot.readied = Some(reference); - slot.ready_by_sender.insert(own, reference); - slot.candidates - .entry(reference) - .or_default() - .readies - .insert(own); - let statement = RbcPhaseStatementV1::Ready { target: reference }; - log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); - self.queue_local_phase(statement); - self.drive_rbc(reference, log); - } - fn queue_local_phase(&mut self, statement: RbcPhaseStatementV1) { if self.pending_phase_set.insert(statement) { self.pending_phases.push_back(statement); @@ -1374,29 +1176,12 @@ impl RbcDagModel { return; } let target = statement.target(); - if !self.committee.known_authority(target.authority) || target.round == 0 { - return; - } - if sender == target.authority - && matches!( - statement, - RbcPhaseStatementV1::Echo { .. } - | RbcPhaseStatementV1::Vote { .. } - | RbcPhaseStatementV1::Ack { .. } - ) - { - // The broadcaster may equivocate. Its stake is deliberately - // excluded from every optimistic certificate phase. - return; - } if sender == self.own_authority { let authorized = self .rbc_slots .get(&(target.round, target.authority)) .is_some_and(|slot| match statement { RbcPhaseStatementV1::Echo { .. } => slot.echoed == Some(target), - RbcPhaseStatementV1::Vote { .. } => slot.voted == Some(target), - RbcPhaseStatementV1::Ack { .. } => slot.acked == Some(target), RbcPhaseStatementV1::Ready { .. } => slot.readied == Some(target), }); if !authorized { @@ -1409,8 +1194,6 @@ impl RbcDagModel { let slot = self.rbc_slot_mut(target); let senders = match statement { RbcPhaseStatementV1::Echo { .. } => &mut slot.echo_by_sender, - RbcPhaseStatementV1::Vote { .. } => &mut slot.vote_by_sender, - RbcPhaseStatementV1::Ack { .. } => &mut slot.ack_by_sender, RbcPhaseStatementV1::Ready { .. } => &mut slot.ready_by_sender, }; match senders.get(&sender) { @@ -1425,12 +1208,6 @@ impl RbcDagModel { RbcPhaseStatementV1::Echo { .. } => { candidate.echoes.insert(sender); } - RbcPhaseStatementV1::Vote { .. } => { - candidate.votes.insert(sender); - } - RbcPhaseStatementV1::Ack { .. } => { - candidate.acks.insert(sender); - } RbcPhaseStatementV1::Ready { .. } => { candidate.readies.insert(sender); } @@ -1438,53 +1215,6 @@ impl RbcDagModel { self.drive_rbc(target, log); } - fn lock_delivery_promise( - &mut self, - target: BlockReference, - basis: DeliveryPromiseBasisV1, - log: &mut TransitionLog, - ) { - if self.delivery_promises.contains_key(&target) { - return; - } - if self - .rbc_slots - .get(&(target.round, target.authority)) - .and_then(|slot| slot.delivered) - .is_some_and(|delivered| delivered != target) - { - // Integrity is enforced locally even though the optimistic and - // fallback quorum proofs already rule out conflicting honest - // deliveries. - return; - } - self.delivery_promises.insert(target, basis); - log.proof(ModelTraceEvent::DeliveryPromiseLocked { target, basis }); - log.effect(ModelEffect::DeliveryPromised(target)); - self.activate_delivery_promise(target, log); - } - - fn activate_delivery_promise(&mut self, target: BlockReference, log: &mut TransitionLog) { - let Some(basis) = self.delivery_promises.get(&target).copied() else { - return; - }; - if basis == DeliveryPromiseBasisV1::Delivered || !self.carriers.contains_key(&target) { - return; - } - let slot = self.rbc_slot_mut(target); - match slot.delivered { - Some(existing) if existing != target => return, - Some(_) => return, - None => slot.delivered = Some(target), - } - self.carriers - .get_mut(&target) - .expect("fast delivery requires exact canonical carrier content") - .delivered = true; - self.pending_delivered_batch_replays.push_back(target); - self.drive_prefix(target.authority, log); - } - fn drive_rbc(&mut self, target: BlockReference, log: &mut TransitionLog) { let slot_key = (target.round, target.authority); if !self @@ -1494,57 +1224,34 @@ impl RbcDagModel { { // Merely staging canonical content is not authenticated RBC // evidence. Candidate state is allocated only by a locally - // authorized phase or an embedded phase statement. + // authorized ECHO or an embedded ECHO/READY statement. return; } - let Some(thresholds) = self.rbc_thresholds(target.authority) else { - return; - }; loop { let header_available = self.carriers.contains_key(&target); - let (echo_stake, vote_stake, ack_stake, ready_stake) = { - let candidate = self - .rbc_slots - .get(&slot_key) - .and_then(|slot| slot.candidates.get(&target)) - .expect("the candidate remains allocated"); - ( - self.voters_stake_excluding(&candidate.echoes, target.authority), - self.voters_stake_excluding(&candidate.votes, target.authority), - self.voters_stake_excluding(&candidate.acks, target.authority), - self.voters_stake(&candidate.readies), - ) - }; - let (vote_trigger, ack_trigger, optimistic_ready_trigger, promise_trigger) = thresholds - .optimistic - .map_or((false, false, false, false), |optimistic| { - ( - echo_stake >= optimistic.vote_from_echo, - echo_stake >= optimistic.converge || vote_stake >= optimistic.converge, - ack_stake >= optimistic.converge, - echo_stake >= optimistic.promise_from_echo, - ) - }); - let ready_trigger = - optimistic_ready_trigger || ready_stake >= thresholds.ready_validity; - let deliver_trigger = ready_stake >= thresholds.ready_quorum; - let promise_missing = !self.delivery_promises.contains_key(&target); - - // A promise names exact canonical content, never a digest learned - // only from phase evidence. - if header_available && promise_missing && promise_trigger { - self.lock_delivery_promise(target, DeliveryPromiseBasisV1::OptimisticEcho, log); - } - let can_send_optimistic_phase = self.own_authority != target.authority; + let q = self.committee.quorum_threshold(); + let v = self.committee.validity_threshold(); let action = { + let echo_stake; + let ready_stake; + { + let slot = self.rbc_slot_mut(target); + let candidate = slot.candidates.entry(target).or_default(); + echo_stake = candidate.echoes.clone(); + ready_stake = candidate.readies.clone(); + } + let echo_stake = self.voters_stake(&echo_stake); + let ready_stake = self.voters_stake(&ready_stake); let slot = self.rbc_slot_mut(target); let candidate = slot.candidates.entry(target).or_default(); + candidate.echo_quorum_observed |= echo_stake >= q; + candidate.ready_validity_observed |= ready_stake >= v; + candidate.ready_quorum_observed |= ready_stake >= q; + let ready_trigger = + candidate.echo_quorum_observed || candidate.ready_validity_observed; let needs_header = !header_available - && ((slot.voted.is_none() && can_send_optimistic_phase && vote_trigger) - || (slot.acked.is_none() && can_send_optimistic_phase && ack_trigger) - || (slot.readied.is_none() && ready_trigger) - || (slot.certified_delivered.is_none() && deliver_trigger) - || (promise_missing && promise_trigger)); + && ((slot.readied.is_none() && ready_trigger) + || (slot.delivered.is_none() && candidate.ready_quorum_observed)); if needs_header { let holders = candidate.holders(); if holders != candidate.requested_holders { @@ -1553,23 +1260,13 @@ impl RbcDagModel { } else { RbcAction::None } - } else if header_available - && slot.voted.is_none() - && can_send_optimistic_phase - && vote_trigger - { - RbcAction::SendVote - } else if header_available - && slot.acked.is_none() - && can_send_optimistic_phase - && ack_trigger - { - RbcAction::SendAck } else if header_available && slot.readied.is_none() && ready_trigger { RbcAction::SendReady - } else if header_available && slot.certified_delivered.is_none() && deliver_trigger + } else if header_available + && slot.delivered.is_none() + && candidate.ready_quorum_observed { - RbcAction::CertifyDelivery + RbcAction::Deliver } else { RbcAction::None } @@ -1588,44 +1285,28 @@ impl RbcDagModel { log.effect(ModelEffect::NeedCarrier { target, holders }); break; } - RbcAction::SendVote => { - let own = self.own_authority; - let slot = self.rbc_slot_mut(target); - slot.voted = Some(target); - slot.vote_by_sender.insert(own, target); - slot.candidates.entry(target).or_default().votes.insert(own); - let statement = RbcPhaseStatementV1::Vote { target }; - log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); - self.queue_local_phase(statement); - } - RbcAction::SendAck => { + RbcAction::SendReady => { let own = self.own_authority; let slot = self.rbc_slot_mut(target); - slot.acked = Some(target); - slot.ack_by_sender.insert(own, target); - slot.candidates.entry(target).or_default().acks.insert(own); - let statement = RbcPhaseStatementV1::Ack { target }; + slot.readied = Some(target); + slot.ready_by_sender.insert(own, target); + slot.candidates + .entry(target) + .or_default() + .readies + .insert(own); + let statement = RbcPhaseStatementV1::Ready { target }; log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); self.queue_local_phase(statement); } - RbcAction::SendReady => { - self.authorize_local_ready(target, log); - } - RbcAction::CertifyDelivery => { - let slot = self.rbc_slot_mut(target); - if slot.delivered.is_some_and(|delivered| delivered != target) { - break; - } - slot.delivered = Some(target); - slot.certified_delivered = Some(target); + RbcAction::Deliver => { + self.rbc_slot_mut(target).delivered = Some(target); let record = self .carriers .get_mut(&target) .expect("delivery requires exact canonical carrier content"); record.delivered = true; - record.certified_delivered = true; log.proof(ModelTraceEvent::DeliveryLocked(target)); - self.lock_delivery_promise(target, DeliveryPromiseBasisV1::Delivered, log); log.effect(ModelEffect::Delivered(target)); self.pending_delivered_batch_replays.push_back(target); self.drive_prefix(target.authority, log); @@ -1909,9 +1590,7 @@ mod tests { .map(|authority| model(Arc::clone(&committee), authority)) .collect(); let mut rounds = Vec::new(); - // ECHO, VOTE/ACK, and READY are embedded in later carriers. Seven - // carrier rounds make the first four rounds mature end-to-end. - for round in 1..=7 { + for round in 1..=6 { rounds.push(run_honest_round(&mut models, round)); } for model in &models { @@ -2260,7 +1939,6 @@ mod tests { assert_eq!(replayed.authenticated_by_slot, live.authenticated_by_slot); assert_eq!(replayed.admitted_by_slot, live.admitted_by_slot); assert_eq!(replayed.rbc_slots, live.rbc_slots); - assert_eq!(replayed.delivery_promises, live.delivery_promises); assert_eq!(replayed.pending_phases, live.pending_phases); assert_eq!(replayed.pending_phase_set, live.pending_phase_set); for reference in references { @@ -2284,12 +1962,10 @@ mod tests { let mut model = model(committee, 0); model.local_carrier_round = 4; let mut queued = Vec::new(); - for round in 1..=2 { + for round in 1..=3 { for author in 0..4 { let target = BlockReference::new_test(author, round); queued.push(RbcPhaseStatementV1::Echo { target }); - queued.push(RbcPhaseStatementV1::Vote { target }); - queued.push(RbcPhaseStatementV1::Ack { target }); queued.push(RbcPhaseStatementV1::Ready { target }); } } @@ -2297,8 +1973,8 @@ mod tests { model.queue_local_phase(*statement); } - assert_eq!(model.pending_phase_backlog_len(), 32); - assert_eq!(model.pending_phase_batch(), queued[..24]); + assert_eq!(model.pending_phase_backlog_len(), 24); + assert_eq!(model.pending_phase_batch(), queued[..16]); } #[test] @@ -2374,545 +2050,6 @@ mod tests { ); } - #[test] - fn local_fix_fast_delivers_exact_content_once_without_q_ready_certificate() { - let committee = committee(4); - let model = model(Arc::clone(&committee), 3); - let (own_prev, weak_parents) = model.local_parent_set().unwrap(); - let carrier = candidate( - &committee, - 3, - 1, - own_prev, - weak_parents, - model.pending_phase_batch(), - 0xF0, - ) - .unwrap(); - let target = carrier.reference(); - let plan = model - .plan_input(ModelInputRecord::LocalCarrierFixed(authenticate_local( - &committee, &carrier, - ))) - .unwrap(); - - let fixed = plan - .trace() - .iter() - .position(|event| *event == ModelTraceEvent::LocalCarrierFixed(target)) - .unwrap(); - let promised = plan - .trace() - .iter() - .position(|event| { - *event - == ModelTraceEvent::DeliveryPromiseLocked { - target, - basis: DeliveryPromiseBasisV1::LocalFixed, - } - }) - .unwrap(); - assert!(fixed < promised); - assert_eq!( - plan.effects() - .iter() - .filter(|effect| **effect == ModelEffect::DeliveryPromised(target)) - .count(), - 1 - ); - - let mut committed = model; - committed.commit_plan(plan).unwrap(); - assert_eq!( - committed.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::LocalFixed) - ); - assert_eq!(committed.delivered(3, 1), Some(target)); - assert_eq!(committed.certified_delivered(3, 1), None); - assert!(committed.lifecycle(&target).unwrap().delivered); - assert!(!committed.lifecycle(&target).unwrap().certified_delivered); - assert!(!committed.lifecycle(&target).unwrap().prefix_closed); - } - - #[test] - fn weighted_thresholds_follow_the_target_author_formula() { - let cases = [ - (vec![1, 1, 1, 1], 0, (1, 2, 3, Some((2, 2, 2)))), - (vec![1, 1, 1, 1, 1, 1, 1], 0, (2, 3, 5, Some((4, 4, 5)))), - (vec![2, 1, 1, 1, 1, 1], 0, (2, 3, 5, Some((3, 3, 3)))), - (vec![1, 2, 2, 2], 0, (2, 3, 5, Some((4, 4, 5)))), - (vec![3, 1, 1, 1], 0, (1, 2, 5, None)), - ]; - - for (stakes, target_author, (fault, validity, quorum, optimistic)) in cases { - let committee = Committee::new_test(stakes); - let model = model(committee, target_author); - let thresholds = model.rbc_thresholds(target_author).unwrap(); - assert_eq!(thresholds.fault, fault); - assert_eq!(thresholds.ready_validity, validity); - assert_eq!(thresholds.ready_quorum, quorum); - assert_eq!( - thresholds.optimistic.map(|thresholds| ( - thresholds.vote_from_echo, - thresholds.converge, - thresholds.promise_from_echo, - )), - optimistic - ); - } - } - - #[test] - fn exhaustive_weighted_echo_subsets_promise_exactly_at_o() { - for stakes in [ - vec![1, 1, 1, 1, 1, 1, 1], - vec![2, 1, 1, 1, 1, 1], - vec![1, 2, 1, 2, 1], - vec![2, 3, 1, 1, 1, 1, 1], - ] { - let committee = Committee::new_test(stakes); - let template = model(Arc::clone(&committee), 0); - let threshold = template - .rbc_thresholds(0) - .unwrap() - .optimistic - .unwrap() - .promise_from_echo; - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A0).unwrap(); - let target = carrier.reference(); - let non_authors: Vec<_> = committee - .authorities() - .filter(|sender| *sender != 0) - .collect(); - - for mask in 0usize..(1usize << non_authors.len()) { - for reverse in [false, true] { - let mut model = model(Arc::clone(&committee), 0); - model.stage_candidate(carrier.clone()).unwrap(); - // The author is never counted, even if it embeds an ECHO. - record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }); - let mut selected: Vec<_> = non_authors - .iter() - .enumerate() - .filter_map(|(index, sender)| ((mask >> index) & 1 == 1).then_some(*sender)) - .collect(); - if reverse { - selected.reverse(); - } - let mut observed_stake = 0; - for sender in selected { - observed_stake += committee.get_stake(sender).unwrap(); - record_phase(&mut model, sender, RbcPhaseStatementV1::Echo { target }); - assert_eq!( - model.delivery_promise_basis(&target).is_some(), - observed_stake >= threshold - ); - } - assert_eq!( - model.delivery_promise_basis(&target), - (observed_stake >= threshold) - .then_some(DeliveryPromiseBasisV1::OptimisticEcho) - ); - } - } - } - } - - #[test] - fn exhaustive_weighted_optimistic_certificates_intersect_honestly() { - for stakes in [ - vec![1, 1, 1, 1, 1, 1, 1], - vec![2, 1, 1, 1, 1, 1], - vec![1, 2, 1, 2, 1], - vec![2, 3, 1, 1, 1, 1, 1], - ] { - let committee = Committee::new_test(stakes); - let model = model(Arc::clone(&committee), 0); - let thresholds = model.rbc_thresholds(0).unwrap(); - let optimistic = thresholds.optimistic.unwrap(); - let author_stake = committee.get_stake(0).unwrap(); - let residual_fault = thresholds.fault - author_stake; - let non_authors: Vec<_> = committee - .authorities() - .filter(|sender| *sender != 0) - .collect(); - let subset_stake = |mask: usize| { - non_authors - .iter() - .enumerate() - .filter(|(index, _)| (mask >> index) & 1 == 1) - .map(|(_, sender)| committee.get_stake(*sender).unwrap()) - .sum::() - }; - let limit = 1usize << non_authors.len(); - let certificates: Vec<_> = (0..limit) - .filter(|mask| subset_stake(*mask) >= optimistic.promise_from_echo) - .collect(); - let byzantine_sets: Vec<_> = (0..limit) - .filter(|mask| subset_stake(*mask) <= residual_fault) - .collect(); - - for first in &certificates { - for second in &certificates { - for byzantine in &byzantine_sets { - // Every pair of selective O certificates shares a - // non-author sender outside the remaining Byzantine - // budget. That honest ECHO lock forbids two values. - assert_ne!(first & second & !byzantine, 0); - } - } - } - } - } - - #[test] - fn weighted_four_phase_rules_and_q_ready_delivery_are_exact() { - let committee = committee(7); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A1).unwrap(); - let target = carrier.reference(); - - let mut from_echo = model(Arc::clone(&committee), 6); - from_echo.stage_candidate(carrier.clone()).unwrap(); - for sender in 1..=3 { - record_phase(&mut from_echo, sender, RbcPhaseStatementV1::Echo { target }); - } - assert!(from_echo.rbc_slots[&(1, 0)].voted.is_none()); - assert!(from_echo.rbc_slots[&(1, 0)].acked.is_none()); - record_phase(&mut from_echo, 4, RbcPhaseStatementV1::Echo { target }); - assert_eq!(from_echo.rbc_slots[&(1, 0)].voted, Some(target)); - assert_eq!(from_echo.rbc_slots[&(1, 0)].acked, Some(target)); - assert_eq!(from_echo.delivery_promise_basis(&target), None); - record_phase(&mut from_echo, 5, RbcPhaseStatementV1::Echo { target }); - assert_eq!( - from_echo.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::OptimisticEcho) - ); - - let mut from_vote = model(Arc::clone(&committee), 6); - from_vote.stage_candidate(carrier.clone()).unwrap(); - for sender in 1..=3 { - record_phase(&mut from_vote, sender, RbcPhaseStatementV1::Vote { target }); - } - assert!(from_vote.rbc_slots[&(1, 0)].acked.is_none()); - record_phase(&mut from_vote, 4, RbcPhaseStatementV1::Vote { target }); - assert_eq!(from_vote.rbc_slots[&(1, 0)].acked, Some(target)); - - let mut from_ack = model(Arc::clone(&committee), 6); - from_ack.stage_candidate(carrier.clone()).unwrap(); - for sender in 1..=3 { - record_phase(&mut from_ack, sender, RbcPhaseStatementV1::Ack { target }); - } - assert!(from_ack.rbc_slots[&(1, 0)].readied.is_none()); - record_phase(&mut from_ack, 4, RbcPhaseStatementV1::Ack { target }); - assert_eq!(from_ack.rbc_slots[&(1, 0)].readied, Some(target)); - - let mut from_ready = model(Arc::clone(&committee), 0); - from_ready.stage_candidate(carrier).unwrap(); - for sender in 1..=2 { - record_phase( - &mut from_ready, - sender, - RbcPhaseStatementV1::Ready { target }, - ); - } - assert!(from_ready.rbc_slots[&(1, 0)].readied.is_none()); - assert_eq!(from_ready.delivered(0, 1), None); - record_phase(&mut from_ready, 3, RbcPhaseStatementV1::Ready { target }); - assert_eq!(from_ready.rbc_slots[&(1, 0)].readied, Some(target)); - // Three remote READYs plus the local READY have stake four, below Q=5. - assert_eq!(from_ready.delivered(0, 1), None); - record_phase(&mut from_ready, 4, RbcPhaseStatementV1::Ready { target }); - assert_eq!(from_ready.delivered(0, 1), Some(target)); - } - - #[test] - fn missing_content_blocks_every_phase_and_requests_all_phase_holders() { - let committee = committee(7); - let mut model = model(Arc::clone(&committee), 6); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A2).unwrap(); - let target = carrier.reference(); - - for sender in 1..=4 { - record_phase(&mut model, sender, RbcPhaseStatementV1::Echo { target }); - } - record_phase(&mut model, 5, RbcPhaseStatementV1::Vote { target }); - record_phase(&mut model, 5, RbcPhaseStatementV1::Ack { target }); - let effects = record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target }); - assert!(matches!( - effects.as_slice(), - [ModelEffect::NeedCarrier { target: requested, holders }] - if *requested == target && holders == &[0, 1, 2, 3, 4, 5] - )); - let slot = &model.rbc_slots[&(1, 0)]; - assert!(slot.voted.is_none()); - assert!(slot.acked.is_none()); - assert!(slot.readied.is_none()); - assert!(slot.delivered.is_none()); - assert_eq!(model.delivery_promise_basis(&target), None); - - model.recover_carrier(carrier).unwrap(); - let slot = &model.rbc_slots[&(1, 0)]; - assert_eq!(slot.voted, Some(target)); - assert_eq!(slot.acked, Some(target)); - } - - #[test] - fn high_stake_honest_author_promises_on_auth_and_seeds_ready_locally() { - let committee = Committee::new_test(vec![3, 1, 1, 1]); - let mut author = model(Arc::clone(&committee), 0); - let (own_prev, weak) = author.local_parent_set().unwrap(); - let carrier = candidate( - &committee, - 0, - 1, - own_prev, - weak, - author.pending_phase_batch(), - 0x1A3, - ) - .unwrap(); - let target = carrier.reference(); - let effects = author - .start_local_carrier(authenticate_local(&committee, &carrier)) - .unwrap(); - assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); - assert_eq!( - author.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::LocalFixed) - ); - assert_eq!(author.rbc_slots[&(1, 0)].readied, Some(target)); - assert!( - author - .pending_phases - .contains(&RbcPhaseStatementV1::Ready { target }) - ); - - let mut receiver = model(Arc::clone(&committee), 3); - receiver.stage_candidate(carrier.clone()).unwrap(); - assert_eq!(receiver.delivery_promise_basis(&target), None); - let effects = receiver - .receive_authenticated(authenticate_for(&committee, &carrier, 3)) - .unwrap(); - assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); - assert_eq!( - receiver.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::HonestAuthor) - ); - assert!(receiver.rbc_slots[&(1, 0)].readied.is_none()); - assert_eq!(receiver.delivered(0, 1), Some(target)); - assert_eq!(receiver.certified_delivered(0, 1), None); - - record_phase(&mut receiver, 0, RbcPhaseStatementV1::Ready { target }); - assert_eq!(receiver.rbc_slots[&(1, 0)].readied, Some(target)); - assert_eq!(receiver.delivered(0, 1), Some(target)); - assert_eq!(receiver.certified_delivered(0, 1), None); - record_phase(&mut receiver, 1, RbcPhaseStatementV1::Ready { target }); - assert_eq!(receiver.delivered(0, 1), Some(target)); - assert_eq!(receiver.certified_delivered(0, 1), Some(target)); - - // Even a test-only forged second author capability cannot bypass the - // receiver's exact authenticated slot lock. - let (own_prev, weak) = genesis_parents(&committee, 0); - let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0x1A4).unwrap(); - let conflicting_ref = conflicting.reference(); - receiver - .receive_authenticated(authenticate_for(&committee, &conflicting, 3)) - .unwrap(); - assert_eq!(receiver.delivery_promise_basis(&conflicting_ref), None); - } - - #[test] - fn raw_q_that_counts_the_target_author_is_not_an_optimistic_promise() { - let committee = committee(7); - let mut model = model(Arc::clone(&committee), 0); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF1).unwrap(); - let target = carrier.reference(); - model.stage_candidate(carrier).unwrap(); - - // W=7, F=2, a=1 gives O=5. A raw Q=5 that includes the - // equivocating target author contains only four admissible ECHOs. - assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }).is_empty()); - for sender in 1..=4 { - assert!( - record_phase(&mut model, sender, RbcPhaseStatementV1::Echo { target }).is_empty() - ); - } - assert_eq!(model.delivery_promise_basis(&target), None); - assert_eq!(model.delivered(0, 1), None); - - let effects = record_phase(&mut model, 5, RbcPhaseStatementV1::Echo { target }); - assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); - assert_eq!( - model.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::OptimisticEcho) - ); - assert_eq!(model.delivered(0, 1), Some(target)); - assert_eq!(model.certified_delivered(0, 1), None); - - // Exact replay and a conflicting later ECHO are both idempotent and - // cannot emit a second promise. - assert!(record_phase(&mut model, 5, RbcPhaseStatementV1::Echo { target }).is_empty()); - let mut conflicting = target; - conflicting.digest = crate::types::BlockDigest::from([0xF2; 32]); - assert!( - record_phase( - &mut model, - 5, - RbcPhaseStatementV1::Echo { - target: conflicting, - }, - ) - .is_empty() - ); - assert_eq!(model.delivery_promises.len(), 1); - } - - #[test] - fn selective_and_equivocating_echoes_cannot_promise_two_values() { - let committee = committee(7); - let mut model = model(Arc::clone(&committee), 0); - let (own_prev, weak) = genesis_parents(&committee, 0); - let first = candidate(&committee, 0, 1, own_prev, weak.clone(), Vec::new(), 0xF3).unwrap(); - let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF4).unwrap(); - let first_ref = first.reference(); - let conflicting_ref = conflicting.reference(); - - model.stage_candidate(first).unwrap(); - model.stage_candidate(conflicting).unwrap(); - for sender in 1..=4 { - record_phase( - &mut model, - sender, - RbcPhaseStatementV1::Echo { target: first_ref }, - ); - } - for sender in 5..=6 { - record_phase( - &mut model, - sender, - RbcPhaseStatementV1::Echo { - target: conflicting_ref, - }, - ); - } - // Sender 1 equivocates after locking the first value. The second - // statement is ignored slot-globally for ECHO. - record_phase( - &mut model, - 1, - RbcPhaseStatementV1::Echo { - target: conflicting_ref, - }, - ); - - assert_eq!( - model.rbc_slots.get(&(1, 0)).unwrap().candidates[&first_ref] - .echoes - .len(), - 4 - ); - assert_eq!( - model.rbc_slots[&(1, 0)].candidates[&conflicting_ref] - .echoes - .len(), - 2 - ); - assert_eq!(model.delivery_promise_basis(&first_ref), None); - assert_eq!(model.delivery_promise_basis(&conflicting_ref), None); - assert!(model.delivery_promises.is_empty()); - } - - #[test] - fn delivered_fallback_promises_before_the_delivered_effect() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF5).unwrap(); - let target = carrier.reference(); - model.stage_candidate(carrier).unwrap(); - - assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target }).is_empty()); - let effects = record_phase(&mut model, 1, RbcPhaseStatementV1::Ready { target }); - assert_eq!( - effects, - vec![ - ModelEffect::DeliveryPromised(target), - ModelEffect::Delivered(target), - ] - ); - assert_eq!( - model.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::Delivered) - ); - } - - #[test] - fn promised_effects_and_locks_replay_deterministically_from_typed_inputs() { - let committee = committee(4); - let context = context(&committee); - let mut live = RbcDagModel::new(Arc::clone(&committee), 3, context).unwrap(); - let (own_prev, weak) = genesis_parents(&committee, 0); - let target_carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 0xF6).unwrap(); - let target = target_carrier.reference(); - let mut records = vec![ModelInputRecord::AuthenticatedIngress(authenticate_for( - &committee, - &target_carrier, - 3, - ))]; - - for author in 0..3 { - let outer = candidate( - &committee, - author, - 2, - BlockReference::new_test(author, 1), - committee - .authorities() - .filter(|other| *other != author) - .take(2) - .map(|other| BlockReference::new_test(other, 1)) - .collect(), - vec![RbcPhaseStatementV1::Echo { target }], - 0xF7 + u64::from(author), - ) - .unwrap(); - records.push(ModelInputRecord::AuthenticatedIngress(authenticate_for( - &committee, &outer, 3, - ))); - } - - let mut live_trace = Vec::new(); - for record in records.iter().cloned() { - let plan = live.plan_input(record).unwrap(); - live_trace.extend_from_slice(plan.trace()); - live.commit_plan(plan).unwrap(); - } - let (replayed, replay_trace) = - RbcDagModel::replay_from_records(committee, 3, context, records).unwrap(); - - assert_eq!(replay_trace, live_trace); - assert_eq!(replayed.delivery_promises, live.delivery_promises); - assert_eq!( - replay_trace - .iter() - .filter(|event| { - **event == ModelTraceEvent::Effect(ModelEffect::DeliveryPromised(target)) - }) - .count(), - 1 - ); - assert_eq!( - replayed.delivery_promise_basis(&target), - Some(DeliveryPromiseBasisV1::OptimisticEcho) - ); - assert_eq!(replayed.delivered(0, 1), Some(target)); - assert_eq!(replayed.certified_delivered(0, 1), None); - } - #[test] fn threshold_before_header_requests_then_recovers_exact_carrier() { let committee = committee(4); @@ -2921,6 +2058,7 @@ mod tests { let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 1).unwrap(); let target = carrier.reference(); + assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }).is_empty()); assert!(record_phase(&mut model, 1, RbcPhaseStatementV1::Echo { target }).is_empty()); assert!(matches!( record_phase( @@ -2930,20 +2068,14 @@ mod tests { ) .as_slice(), [ModelEffect::NeedCarrier { target: requested, holders }] - if *requested == target && holders == &[1, 2] + if *requested == target && holders == &[0, 1, 2] )); - let effects = model.recover_carrier(carrier).unwrap(); - assert_eq!(effects, vec![ModelEffect::DeliveryPromised(target)]); - assert!( - model - .pending_phases - .contains(&RbcPhaseStatementV1::Vote { target }) - ); + model.recover_carrier(carrier).unwrap(); assert!( model .pending_phases - .contains(&RbcPhaseStatementV1::Ack { target }) + .contains(&RbcPhaseStatementV1::Ready { target }) ); let lifecycle = model.lifecycle(&target).unwrap(); assert!(!lifecycle.authenticated); @@ -3088,8 +2220,6 @@ mod tests { let target = BlockReference::new_test(0, 1); assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Echo { target }).is_empty()); - assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Vote { target }).is_empty()); - assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Ack { target }).is_empty()); assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Ready { target }).is_empty()); assert!(model.rbc_slots.is_empty()); } @@ -3131,8 +2261,8 @@ mod tests { [ModelEffect::NeedCarrier { target, .. }] if *target == first )); let slot = model.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.echo_by_sender.len(), 2); - assert_eq!(slot.echo_by_sender[&1], first); + assert_eq!(slot.echo_by_sender.len(), 3); + assert_eq!(slot.echo_by_sender[&0], first); assert!(!slot.candidates.contains_key(&conflicting)); record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target: first }); @@ -3146,26 +2276,6 @@ mod tests { let slot = model.rbc_slots.get(&(1, 0)).unwrap(); assert_eq!(slot.ready_by_sender[&0], first); assert!(!slot.candidates.contains_key(&conflicting)); - - for statement in [ - RbcPhaseStatementV1::Vote { target: first }, - RbcPhaseStatementV1::Ack { target: first }, - ] { - record_phase(&mut model, 1, statement); - let conflicting_statement = match statement { - RbcPhaseStatementV1::Vote { .. } => RbcPhaseStatementV1::Vote { - target: conflicting, - }, - RbcPhaseStatementV1::Ack { .. } => RbcPhaseStatementV1::Ack { - target: conflicting, - }, - _ => unreachable!(), - }; - record_phase(&mut model, 1, conflicting_statement); - } - let slot = model.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.vote_by_sender[&1], first); - assert_eq!(slot.ack_by_sender[&1], first); } #[test] @@ -3299,7 +2409,7 @@ mod tests { assert_eq!(receiver.delivered(0, 1), Some(first_ref)); assert_ne!(receiver.delivered(0, 1), Some(conflicting_ref)); let slot = receiver.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.echo_by_sender.get(&0), None); + assert_eq!(slot.echo_by_sender.get(&0), Some(&first_ref)); assert!(!slot.candidates.contains_key(&conflicting_ref)); } @@ -3526,14 +2636,14 @@ mod tests { let carrier = candidate( &committee, 1, - 66, - BlockReference::new_test(1, 65), + 6, + BlockReference::new_test(1, 5), vec![ - BlockReference::new_test(0, 65), - BlockReference::new_test(2, 65), + BlockReference::new_test(0, 5), + BlockReference::new_test(2, 5), ], Vec::new(), - 660, + 60, ) .unwrap(); let reference = carrier.reference(); @@ -3543,8 +2653,8 @@ mod tests { model.receive_authenticated(authenticated), Err(ModelError::FutureCarrierOutsideBuffer { current: 1, - maximum: 65, - actual: 66, + maximum: 5, + actual: 6, }) ); assert!(model.lifecycle(&reference).is_none()); @@ -3552,47 +2662,6 @@ mod tests { assert!(model.rbc_slots.is_empty()); } - #[test] - fn carrier_at_the_future_buffer_boundary_is_retained_but_cannot_advance() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 0); - let carrier = candidate( - &committee, - 1, - 65, - BlockReference::new_test(1, 64), - vec![ - BlockReference::new_test(0, 64), - BlockReference::new_test(2, 64), - ], - Vec::new(), - 650, - ) - .unwrap(); - let reference = carrier.reference(); - - assert!( - model - .receive_authenticated(authenticate_for(&committee, &carrier, 0)) - .unwrap() - .is_empty() - ); - assert_eq!(model.local_carrier_round(), 1); - assert_eq!( - model.lifecycle(&reference), - Some(CarrierLifecycle { - authenticated: true, - admitted: false, - phase_batch_processed: true, - delivered: false, - certified_delivered: false, - data_available: false, - prefix_closed: false, - }) - ); - assert!(model.rbc_slots.is_empty()); - } - #[test] fn buffered_authenticated_carrier_is_promoted_when_window_opens() { let committee = committee(4); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 71950faa..5c17e5ee 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -9,7 +9,6 @@ //! evaluates the explicit vote/no-vote evidence committed by those vertices. use std::{ - cmp::Reverse, collections::{BTreeMap, BTreeSet}, error::Error, fmt, @@ -57,27 +56,6 @@ pub enum ProjectionDecisionV1 { }, } -/// Immutable level-two evidence that allows an honest author to create the -/// next consensus vertex through the optimistic C1 pacemaker condition. -/// -/// Every returned reference is from consensus round `c - 1` and there is at -/// most one reference per author. A vote witness contains quorum stake voting -/// for one exact leader at `c - 2`, plus any caller-required own/leader parent -/// that is not already in that proof. A skip witness is the deterministic -/// union of the per-candidate negative-choice quorums required by the -/// direct-skip evaluator and those same required parents. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum C1StrongParentWitnessV1 { - Vote { - leader: ConsensusVertexReference, - parents: Vec, - }, - DirectSkip { - slot: LeaderSlotV1, - parents: Vec, - }, -} - #[derive(Clone, Debug, Eq, PartialEq)] pub enum CertifiedProjectionError { CommitteeMismatch, @@ -111,10 +89,16 @@ pub enum CertifiedProjectionError { required: Option, actual: Option, }, + FrontierRegressesCommitted { + authority: AuthorityIndex, + committed: Option, + actual: Option, + }, StakeOverflow, InvalidLeaderSlot(LeaderSlotV1), MultipleCertifiedLeaderValues(LeaderSlotV1), ConflictingDirectDecision(LeaderSlotV1), + AnchorNotCommitted(ConsensusVertexReference), AnchorTooEarly { slot: LeaderSlotV1, anchor: ConsensusVertexReference, @@ -257,12 +241,6 @@ impl CertifiedProjectionModel { Ok(()) } - pub(crate) fn is_data_available(&self, reference: BlockReference) -> bool { - self.carriers - .get(&reference) - .is_some_and(|state| state.data_available) - } - pub fn carrier_is_stored(&self, reference: BlockReference) -> bool { self.carriers.contains_key(&reference) } @@ -325,15 +303,6 @@ impl CertifiedProjectionModel { self.vertices.len() } - pub(crate) fn projected_stake_at_round(&self, round: RoundNumber) -> Stake { - self.vertices_at_round(round) - .map(|(reference, _)| reference.author()) - .collect::>() - .into_iter() - .filter_map(|authority| self.committee.get_stake(authority)) - .fold(0, Stake::saturating_add) - } - pub fn slot_values( &self, author: AuthorityIndex, @@ -360,122 +329,6 @@ impl CertifiedProjectionModel { .map(|projected| projected.vertex.leader_choice()) } - /// Return the exact C1 witness for creating consensus round `c`. - /// - /// Exact-vote witnesses consider every projected equivocation, then select - /// one deterministic matching value per author. Direct-skip witnesses use - /// one deterministic representative per author so their per-candidate - /// negative quorums have one compatible immutable union. Returning `None` - /// means C1 is not ready and the caller must wait or use a separately - /// justified C2/C3 fallback. - pub(crate) fn c1_strong_parent_witness( - &self, - consensus_round: RoundNumber, - required_parents: &[ConsensusVertexReference], - ) -> Result, CertifiedProjectionError> { - if consensus_round < 3 { - return Ok(None); - } - let voting_round = consensus_round - 1; - let slot = self.leader_slot(consensus_round - 2); - let representatives = self.deterministic_round_values(voting_round); - - let mut votes = BTreeMap::< - ConsensusVertexReference, - BTreeMap, - >::new(); - for (reference, _) in self.vertices_at_round(voting_round) { - if let Some(LeaderChoiceV1::Vote { leader }) = self.leader_choice(reference) { - votes - .entry(leader) - .or_default() - .entry(reference.author()) - .or_insert(reference); - } - } - let mut vote_witnesses = Vec::new(); - for (leader, voters) in votes { - if let Some(parents) = - self.frontier_fresh_quorum(voters.into_values(), required_parents)? - { - vote_witnesses.push((leader, parents)); - } - } - if vote_witnesses.len() > 1 { - return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues( - slot, - )); - } - if let Some((leader, parents)) = vote_witnesses.pop() { - return Ok(Some(C1StrongParentWitnessV1::Vote { leader, parents })); - } - - let candidates = self.slot_values(slot.author, slot.round); - let mut union = BTreeMap::new(); - if candidates.is_empty() { - let Some(parents) = - self.frontier_fresh_quorum(representatives.values().copied(), required_parents)? - else { - return Ok(None); - }; - for reference in parents { - union.insert(reference.author(), reference); - } - } else { - for candidate in candidates { - let negative = representatives.values().copied().filter(|reference| { - self.leader_choice(*reference) - .is_some_and(|choice| match choice { - LeaderChoiceV1::Vote { leader } => leader != candidate, - LeaderChoiceV1::NoVote { .. } => true, - }) - }); - let Some(parents) = self.lexicographic_quorum(negative)? else { - return Ok(None); - }; - for reference in parents { - union.insert(reference.author(), reference); - } - } - if !self.extend_with_required_parents(&mut union, required_parents) { - return Ok(None); - } - } - Ok(Some(C1StrongParentWitnessV1::DirectSkip { - slot, - parents: union.into_values().collect(), - })) - } - - /// Componentwise join of the exact effective frontiers inherited through - /// one immutable strong-parent set. Callers extend only their own - /// component from this base; copying the globally freshest closed - /// frontier would make every consensus vertex wait for unrelated - /// all-author delivery tails. - pub(crate) fn joined_strong_parent_frontier( - &self, - strong_parents: &[ConsensusVertexReference], - ) -> Result { - let mut parent_frontiers = Vec::with_capacity(strong_parents.len()); - for parent in strong_parents { - if parent.consensus_round() == 0 { - if parent.carrier() != carrier_genesis_reference(parent.author()) { - return Err(CertifiedProjectionError::InvalidGenesisStrongParent( - *parent, - )); - } - parent_frontiers.push(vec![None; self.committee.len()]); - continue; - } - let projected = self - .vertices - .get(parent) - .ok_or(CertifiedProjectionError::MissingStrongParent(*parent))?; - parent_frontiers.push(projected.effective_frontier.clone()); - } - self.join_frontiers(&parent_frontiers) - } - /// Project one optional consensus vertex if every stateful eligibility /// condition holds. Failure leaves the enclosing carrier untouched. pub fn try_project( @@ -636,44 +489,34 @@ impl CertifiedProjectionModel { } } - /// Record an externally selected committed anchor and return the monotone - /// componentwise join of every committed anchor frontier. Consecutive - /// Starfish leaders need not be ancestors of one another, so their exact - /// frontiers may advance different authority components concurrently. - /// Logical anchor membership is therefore independent of whether this - /// particular anchor advances the accumulated output frontier. + /// Record an externally selected committed anchor while enforcing exact + /// componentwise frontier monotonicity. The runtime committer remains out + /// of scope for this model. pub fn record_committed_anchor( &mut self, anchor: ConsensusVertexReference, - ) -> Result { + ) -> Result<(), CertifiedProjectionError> { let projected = self .vertices .get(&anchor) .ok_or(CertifiedProjectionError::MissingStrongParent(anchor))?; let frontier = projected.effective_frontier.clone(); - let accumulated = self.committed_frontier.clone(); - let joined = self.join_frontiers(&[accumulated, frontier])?; - self.committed_frontier.clone_from(&joined); + self.ensure_dominates_committed(&frontier)?; + self.committed_frontier = frontier; self.committed_anchors.insert(anchor); - Ok(joined) + Ok(()) } - /// Decide an older leader from a later projected anchor selected by the - /// ordered committer. A reachable certifying-round vertex with a QC yields - /// commit; absence yields skip. - /// - /// The anchor's frontier is deliberately not required to have been - /// applied yet. The committer first derives the finalized leader sequence - /// from newest to oldest, then applies committed frontiers in the opposite - /// (oldest-to-newest) order. + /// Decide an older leader from a later committed anchor. A reachable + /// certifying-round vertex with a QC yields commit; absence yields skip. pub fn indirect_decision( &self, slot: LeaderSlotV1, anchor: ConsensusVertexReference, ) -> Result { self.validate_leader_slot(slot)?; - if !self.vertices.contains_key(&anchor) { - return Err(CertifiedProjectionError::MissingStrongParent(anchor)); + if !self.committed_anchors.contains(&anchor) { + return Err(CertifiedProjectionError::AnchorNotCommitted(anchor)); } let minimum_anchor_round = slot.round.saturating_add(3); if anchor.consensus_round() < minimum_anchor_round { @@ -795,65 +638,31 @@ impl CertifiedProjectionModel { Ok(()) } - /// True iff `descendant` is the same exact prefix tip as `base`, or an - /// exact self-chain extension whose intermediate carrier headers are known. - fn is_exact_extension( - &self, - base: Option, - descendant: Option, - ) -> bool { - self.exact_extension_on_closed_prefix(base, descendant) - .unwrap_or_else(|| self.is_exact_extension_by_chain(base, descendant)) - } - - /// Resolve comparisons whose answer is already encoded by the exact - /// per-author closed prefix. Returning `None` preserves the historical - /// header-chain walk for staged forks and incomplete/non-closed tails. - fn exact_extension_on_closed_prefix( + fn ensure_dominates_committed( &self, - base: Option, - descendant: Option, - ) -> Option { - let Some(descendant) = descendant else { - return Some(base.is_none()); - }; - if base.is_some_and(|base| base.authority != descendant.authority) { - return Some(false); - } - let base_round = base.map_or(0, |reference| reference.round); - if descendant.round < base_round { - return Some(false); - } - if base == Some(descendant) { - return Some(true); - } - if descendant.round == base_round { - return Some(match base { - Some(_) => false, - None => descendant == carrier_genesis_reference(descendant.authority), - }); - } - if !self.is_exact_closed_prefix_reference(descendant) { - return None; - } - match base { - None => Some(true), - Some(base) if self.is_exact_closed_prefix_reference(base) => Some(true), - Some(_) => None, - } - } - - fn is_exact_closed_prefix_reference(&self, reference: BlockReference) -> bool { - if reference.round == 0 { - return reference == carrier_genesis_reference(reference.authority); + frontier: &[Option], + ) -> Result<(), CertifiedProjectionError> { + for (index, (committed, actual)) in self + .committed_frontier + .iter() + .copied() + .zip(frontier.iter().copied()) + .enumerate() + { + if !self.is_exact_extension(committed, actual) { + return Err(CertifiedProjectionError::FrontierRegressesCommitted { + authority: index as AuthorityIndex, + committed, + actual, + }); + } } - self.closed_prefixes - .get(reference.authority as usize) - .and_then(|prefix| prefix.get((reference.round - 1) as usize)) - .is_some_and(|closed| *closed == reference) + Ok(()) } - fn is_exact_extension_by_chain( + /// True iff `descendant` is the same exact prefix tip as `base`, or an + /// exact self-chain extension whose intermediate carrier headers are known. + fn is_exact_extension( &self, base: Option, descendant: Option, @@ -908,215 +717,6 @@ impl CertifiedProjectionModel { }) } - fn deterministic_round_values( - &self, - round: RoundNumber, - ) -> BTreeMap { - let mut by_author = BTreeMap::new(); - for (reference, _) in self.vertices_at_round(round) { - by_author.entry(reference.author()).or_insert(reference); - } - by_author - } - - fn lexicographic_quorum( - &self, - references: impl Iterator, - ) -> Result>, CertifiedProjectionError> { - let mut stake = 0u64; - let mut selected = Vec::new(); - for reference in references { - let author_stake = self - .committee - .get_stake(reference.author()) - .ok_or(CertifiedProjectionError::StakeOverflow)?; - stake = stake - .checked_add(author_stake) - .ok_or(CertifiedProjectionError::StakeOverflow)?; - selected.push(reference); - if stake >= self.committee.quorum_threshold() { - return Ok(Some(selected)); - } - } - Ok(None) - } - - /// Choose quorum evidence that maximizes new exact-prefix coverage without - /// increasing the parent budget of the former canonical-prefix selector. - /// - /// The baseline is the lexicographic quorum unioned with every required - /// parent. The greedy candidate set may use at most that many references. - /// Each choice must leave enough remaining slots to complete weighted - /// quorum stake; if freshness selection cannot do so, the known-valid - /// baseline is returned. Required references count toward proof stake only - /// when they are exact members of `references`. - pub(crate) fn frontier_fresh_quorum( - &self, - references: impl IntoIterator, - required: &[ConsensusVertexReference], - ) -> Result>, CertifiedProjectionError> { - let mut candidates = BTreeMap::::new(); - for reference in references { - candidates - .entry(reference.author()) - .and_modify(|existing| *existing = (*existing).min(reference)) - .or_insert(reference); - } - - let Some(baseline_proof) = self.lexicographic_quorum(candidates.values().copied())? else { - return Ok(None); - }; - let mut baseline = baseline_proof - .into_iter() - .map(|reference| (reference.author(), reference)) - .collect::>(); - if !self.extend_with_required_parents(&mut baseline, required) { - return Ok(None); - } - let parent_budget = baseline.len(); - - let mut selected = BTreeMap::new(); - if !self.extend_with_required_parents(&mut selected, required) { - return Ok(None); - } - let mut proof_stake = selected - .iter() - .filter(|(author, reference)| candidates.get(author) == Some(reference)) - .try_fold(0u64, |stake, (author, _)| { - stake - .checked_add( - self.committee - .get_stake(*author) - .ok_or(CertifiedProjectionError::StakeOverflow)?, - ) - .ok_or(CertifiedProjectionError::StakeOverflow) - })?; - let quorum = self.committee.quorum_threshold(); - let tie_origin: RoundNumber = candidates - .values() - .next() - .map_or(0, |reference| reference.consensus_round()) - % u32::try_from(self.committee.len()) - .unwrap_or(u32::MAX) - .max(1); - - while proof_stake < quorum && selected.len() < parent_budget { - let selected_references = selected.values().copied().collect::>(); - let base = match self.joined_strong_parent_frontier(&selected_references) { - Ok(base) => base, - Err(_) => return Ok(Some(baseline.into_values().collect())), - }; - let remaining_slots = parent_budget.saturating_sub(selected.len() + 1); - let mut best = None; - for (author, reference) in &candidates { - if selected.contains_key(author) { - continue; - } - let Some(projected) = self.vertices.get(reference) else { - continue; - }; - if self - .join_frontiers(&[base.clone(), projected.effective_frontier.clone()]) - .is_err() - { - continue; - } - let author_stake = self - .committee - .get_stake(*author) - .ok_or(CertifiedProjectionError::StakeOverflow)?; - let candidate_stake = proof_stake - .checked_add(author_stake) - .ok_or(CertifiedProjectionError::StakeOverflow)?; - let mut remaining_stakes: Vec = candidates - .keys() - .filter(|candidate_author| { - **candidate_author != *author && !selected.contains_key(*candidate_author) - }) - .map(|candidate_author| { - self.committee - .get_stake(*candidate_author) - .ok_or(CertifiedProjectionError::StakeOverflow) - }) - .collect::, _>>()?; - remaining_stakes.sort_unstable_by(|left: &Stake, right: &Stake| right.cmp(left)); - let maximum_completion = remaining_stakes - .into_iter() - .take(remaining_slots) - .try_fold(candidate_stake, |stake: Stake, next: Stake| { - stake - .checked_add(next) - .ok_or(CertifiedProjectionError::StakeOverflow) - })?; - if maximum_completion < quorum { - continue; - } - - let (advanced_components, round_advance) = base - .iter() - .zip(&projected.effective_frontier) - .fold((0usize, 0u64), |(components, advance), (base, tip)| { - let base_round = base.map_or(0, |reference| reference.round); - let tip_round = tip.map_or(0, |reference| reference.round); - let delta = tip_round.saturating_sub(base_round); - ( - components.saturating_add(usize::from(delta != 0)), - advance.saturating_add(u64::from(delta)), - ) - }); - let committee_size = u32::try_from(self.committee.len()) - .unwrap_or(u32::MAX) - .max(1); - let rotating_rank = - u32::from(*author).wrapping_add(committee_size - tie_origin) % committee_size; - let score = ( - advanced_components, - round_advance, - Reverse(rotating_rank), - Reverse(*reference), - ); - if best - .as_ref() - .is_none_or(|(best_score, _, _)| score > *best_score) - { - best = Some((score, *author, *reference)); - } - } - let Some((_, author, reference)) = best else { - return Ok(Some(baseline.into_values().collect())); - }; - selected.insert(author, reference); - proof_stake = proof_stake - .checked_add( - self.committee - .get_stake(author) - .ok_or(CertifiedProjectionError::StakeOverflow)?, - ) - .ok_or(CertifiedProjectionError::StakeOverflow)?; - } - - if proof_stake < quorum { - return Ok(Some(baseline.into_values().collect())); - } - Ok(Some(selected.into_values().collect())) - } - - fn extend_with_required_parents( - &self, - selected: &mut BTreeMap, - required: &[ConsensusVertexReference], - ) -> bool { - for reference in required { - if selected - .insert(reference.author(), *reference) - .is_some_and(|existing| existing != *reference) - { - return false; - } - } - true - } - fn voter_authors( &self, slot: LeaderSlotV1, @@ -1213,7 +813,7 @@ impl CertifiedProjectionModel { } #[cfg(test)] - pub(crate) fn inject_projected_for_test( + fn inject_projected_for_test( &mut self, reference: ConsensusVertexReference, strong_parents: Vec, @@ -1243,139 +843,6 @@ impl CertifiedProjectionModel { } } -/// Independent planning-only projection over promised, data-available carrier -/// prefixes. -/// -/// This wrapper deliberately exposes no decision, committed-anchor, or -/// committed-frontier API. Its inner model reuses the exact carrier-chain, -/// strong-parent, and frontier validation of the certified projection, but -/// interprets that private plane's delivery latch as `DeliveryPromised`. -/// Consequently its contiguous prefixes and projected vertices can run ahead -/// of certification without becoming output authority. -#[derive(Clone)] -pub(crate) struct PromisedProjectionModel { - inner: CertifiedProjectionModel, -} - -impl PromisedProjectionModel { - pub(crate) fn from_committee_context(committee: RbcDagCommitteeContextV1) -> Self { - Self { - inner: CertifiedProjectionModel::from_committee_context(committee), - } - } - - pub(crate) fn stage_carrier( - &mut self, - candidate: CandidateCarrierV1, - ) -> Result<(), CertifiedProjectionError> { - self.inner.stage_carrier(candidate) - } - - /// Mark one exact carrier promised. This advances only the planner's - /// private contiguous prefix once local DA is also established. - pub(crate) fn mark_promised( - &mut self, - reference: BlockReference, - ) -> Result<(), CertifiedProjectionError> { - self.inner.mark_delivered(reference) - } - - pub(crate) fn mark_data_available( - &mut self, - reference: BlockReference, - ) -> Result<(), CertifiedProjectionError> { - self.inner.mark_data_available(reference) - } - - #[cfg(test)] - pub(crate) fn is_data_available(&self, reference: BlockReference) -> bool { - self.inner.is_data_available(reference) - } - - pub(crate) fn carrier_is_stored(&self, reference: BlockReference) -> bool { - self.inner.carrier_is_stored(reference) - } - - #[cfg(test)] - pub(crate) fn promised_tip(&self, authority: AuthorityIndex) -> Option { - self.inner.closed_tip(authority) - } - - #[cfg(test)] - pub(crate) fn is_projected(&self, reference: ConsensusVertexReference) -> bool { - self.inner.is_projected(reference) - } - - #[cfg(test)] - pub(crate) fn projected_vertex( - &self, - reference: ConsensusVertexReference, - ) -> Option<&ConsensusVertexV1> { - self.inner.projected_vertex(reference) - } - - #[cfg(test)] - pub(crate) fn effective_frontier( - &self, - reference: ConsensusVertexReference, - ) -> Option<&[Option]> { - self.inner.effective_frontier(reference) - } - - pub(crate) fn projected_values_at_round( - &self, - round: RoundNumber, - ) -> Vec { - self.inner.projected_values_at_round(round) - } - - pub(crate) fn projected_stake_at_round(&self, round: RoundNumber) -> Stake { - self.inner.projected_stake_at_round(round) - } - - pub(crate) fn c1_strong_parent_witness( - &self, - consensus_round: RoundNumber, - required_parents: &[ConsensusVertexReference], - ) -> Result, CertifiedProjectionError> { - self.inner - .c1_strong_parent_witness(consensus_round, required_parents) - } - - pub(crate) fn frontier_fresh_quorum( - &self, - references: impl IntoIterator, - required: &[ConsensusVertexReference], - ) -> Result>, CertifiedProjectionError> { - self.inner.frontier_fresh_quorum(references, required) - } - - pub(crate) fn joined_strong_parent_frontier( - &self, - strong_parents: &[ConsensusVertexReference], - ) -> Result { - self.inner.joined_strong_parent_frontier(strong_parents) - } - - pub(crate) fn try_project( - &mut self, - carrier_reference: BlockReference, - ) -> Result { - self.inner.try_project(carrier_reference) - } - - #[cfg(test)] - pub(crate) fn inject_projected_for_test( - &mut self, - reference: ConsensusVertexReference, - strong_parents: Vec, - leader_choice: LeaderChoiceV1, - ) { - self.inner - .inject_projected_for_test(reference, strong_parents, leader_choice); - } -} - #[cfg(test)] mod tests { use super::*; @@ -1456,57 +923,6 @@ mod tests { reference } - fn close_author_history( - model: &mut CertifiedProjectionModel, - author: AuthorityIndex, - rounds: RoundNumber, - ) -> Vec { - let mut result = Vec::with_capacity(rounds as usize); - let mut own_previous = carrier_genesis_reference(author); - for round in 1..=rounds { - let previous = model - .committee - .authorities() - .map(|authority| { - if authority == author { - own_previous - } else if round == 1 { - carrier_genesis_reference(authority) - } else { - reference( - authority, - round - 1, - (authority as u8).wrapping_mul(31).wrapping_add(round as u8), - ) - } - }) - .collect::>(); - let carrier = candidate( - &model.committee, - author, - round, - &previous, - None, - (author as u8).wrapping_mul(53).wrapping_add(round as u8), - ); - own_previous = clean(model, carrier); - result.push(own_previous); - } - result - } - - fn assert_exact_extension_matches_chain_oracle( - model: &CertifiedProjectionModel, - base: Option, - descendant: Option, - ) { - assert_eq!( - model.is_exact_extension(base, descendant), - model.is_exact_extension_by_chain(base, descendant), - "base={base:?}, descendant={descendant:?}", - ); - } - fn first_consensus_round( model: &mut CertifiedProjectionModel, ) -> (Vec, Vec) { @@ -1593,229 +1009,6 @@ mod tests { assert_eq!(&effective[1..], expected.as_slice()); } - #[test] - fn closed_prefix_exact_extension_fast_path_matches_chain_oracle() { - const LAST_ROUND: RoundNumber = 512; - - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let closed = close_author_history(&mut model, 0, LAST_ROUND); - let genesis = carrier_genesis_reference(0); - let closed_samples = [closed[0], closed[16], closed[255], closed[511]]; - - assert_eq!( - model.exact_extension_on_closed_prefix(None, Some(closed[511])), - Some(true) - ); - assert_eq!( - model.exact_extension_on_closed_prefix(Some(closed[16]), Some(closed[511])), - Some(true) - ); - for base in [None, Some(genesis)] - .into_iter() - .chain(closed_samples.into_iter().map(Some)) - { - for descendant in [None, Some(genesis)] - .into_iter() - .chain(closed_samples.into_iter().map(Some)) - { - assert_exact_extension_matches_chain_oracle(&model, base, descendant); - } - } - - let same_round_fork = reference(0, LAST_ROUND, 0xF1); - let cross_author = reference(1, LAST_ROUND, 0xF2); - let missing_descendant = reference(0, LAST_ROUND + 4, 0xF3); - let invalid_genesis = reference(0, 0, 0xF4); - for (base, descendant) in [ - (Some(closed[511]), Some(same_round_fork)), - (Some(closed[16]), Some(same_round_fork)), - (Some(closed[16]), Some(cross_author)), - (Some(closed[511]), Some(missing_descendant)), - (None, Some(invalid_genesis)), - (Some(closed[511]), None), - (None, None), - ] { - assert_exact_extension_matches_chain_oracle(&model, base, descendant); - } - - // A fully staged non-closed tail remains an exact known extension and - // therefore exercises the historical chain-walk fallback. - let staged_previous = model - .committee - .authorities() - .map(|authority| { - if authority == 0 { - closed[511] - } else { - reference(authority, LAST_ROUND, 0xA0 + authority as u8) - } - }) - .collect::>(); - let staged = candidate( - &model.committee, - 0, - LAST_ROUND + 1, - &staged_previous, - None, - 0xF5, - ); - let staged_reference = staged.reference(); - model.stage_carrier(staged).unwrap(); - let mut staged_child_previous = staged_previous; - staged_child_previous[0] = staged_reference; - for (authority, previous) in staged_child_previous.iter_mut().enumerate().skip(1) { - *previous = reference( - authority as AuthorityIndex, - LAST_ROUND + 1, - 0xB0 + authority as u8, - ); - } - let staged_child = candidate( - &model.committee, - 0, - LAST_ROUND + 2, - &staged_child_previous, - None, - 0xF6, - ); - let staged_child_reference = staged_child.reference(); - model.stage_carrier(staged_child).unwrap(); - assert_eq!( - model - .exact_extension_on_closed_prefix(Some(closed[511]), Some(staged_child_reference),), - None - ); - assert_exact_extension_matches_chain_oracle( - &model, - Some(closed[511]), - Some(staged_child_reference), - ); - assert!(model.is_exact_extension(Some(closed[511]), Some(staged_child_reference))); - - // A staged descendant whose intermediate own-prev is absent also - // falls back, then rejects exactly as the prior implementation did. - let missing_previous = reference(0, LAST_ROUND + 2, 0xF7); - let missing_chain_previous = model - .committee - .authorities() - .map(|authority| { - if authority == 0 { - missing_previous - } else { - reference(authority, LAST_ROUND + 2, 0xC0 + authority as u8) - } - }) - .collect::>(); - let missing_chain = candidate( - &model.committee, - 0, - LAST_ROUND + 3, - &missing_chain_previous, - None, - 0xF8, - ); - let missing_chain_reference = missing_chain.reference(); - model.stage_carrier(missing_chain).unwrap(); - assert_exact_extension_matches_chain_oracle( - &model, - Some(closed[511]), - Some(missing_chain_reference), - ); - assert!(!model.is_exact_extension(Some(closed[511]), Some(missing_chain_reference))); - } - - #[test] - fn closed_prefix_fast_path_preserves_join_dominance_and_committed_frontiers() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let histories = (0..4) - .map(|authority| close_author_history(&mut model, authority, 32)) - .collect::>(); - let first = vec![ - Some(histories[0][7]), - None, - Some(histories[2][11]), - Some(histories[3][3]), - ]; - let second = vec![ - Some(histories[0][15]), - Some(histories[1][5]), - Some(histories[2][2]), - None, - ]; - let joined = vec![ - Some(histories[0][15]), - Some(histories[1][5]), - Some(histories[2][11]), - Some(histories[3][3]), - ]; - - assert_eq!( - model - .join_frontiers(&[first.clone(), second.clone()]) - .unwrap(), - joined - ); - model.ensure_dominates_parent(&joined, &first).unwrap(); - model.ensure_dominates_parent(&joined, &second).unwrap(); - assert!(matches!( - model.ensure_dominates_parent(&first, &second), - Err(CertifiedProjectionError::FrontierDoesNotDominateParent { authority: 0, .. }) - )); - - let first_anchor = consensus_reference(0, 40, 0xD1); - let second_anchor = consensus_reference(1, 41, 0xD2); - for anchor in [first_anchor, second_anchor] { - model.inject_projected_for_test( - anchor, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 39, - }, - ); - } - model - .vertices - .get_mut(&first_anchor) - .unwrap() - .effective_frontier - .clone_from(&first); - model - .vertices - .get_mut(&second_anchor) - .unwrap() - .effective_frontier - .clone_from(&second); - assert_eq!( - model - .joined_strong_parent_frontier(&[first_anchor, second_anchor]) - .unwrap(), - joined - ); - assert_eq!(model.record_committed_anchor(first_anchor).unwrap(), first); - assert_eq!( - model.record_committed_anchor(second_anchor).unwrap(), - joined - ); - assert!(model.committed_frontier_dominates(&first)); - assert!(model.committed_frontier_dominates(&second)); - assert!(model.committed_frontier_dominates(&joined)); - - let fork = Some(reference(2, histories[2][11].round, 0xD3)); - let mut forked = joined.clone(); - forked[2] = fork; - assert!(matches!( - model.join_frontiers(&[joined.clone(), forked.clone()]), - Err(CertifiedProjectionError::ParentFrontierFork { authority: 2, .. }) - )); - assert!(!model.committed_frontier_dominates(&forked)); - let mut beyond_commit = joined; - beyond_commit[3] = Some(histories[3][31]); - assert!(!model.committed_frontier_dominates(&beyond_commit)); - } - #[test] fn omission_and_same_round_fork_do_not_pass_frontier_checks() { let committee = Committee::new_test(vec![1; 4]); @@ -1913,7 +1106,7 @@ mod tests { } #[test] - fn concurrent_committed_anchor_frontiers_accumulate_by_exact_component_join() { + fn late_vertex_remains_visible_but_cannot_become_a_regressing_anchor() { let committee = Committee::new_test(vec![1; 4]); let mut model = CertifiedProjectionModel::new(committee).unwrap(); let (carriers, parents) = first_consensus_round(&mut model); @@ -1945,23 +1138,12 @@ mod tests { let regressing_carrier = clean(&mut model, regressing); let regressing_vertex = model.try_project(regressing_carrier).unwrap(); assert!(model.is_projected(regressing_vertex)); - let accumulated = model.record_committed_anchor(regressing_vertex).unwrap(); - assert_eq!( - accumulated, - vec![ - Some(anchor_carrier), - Some(carriers[1]), - Some(regressing_carrier), - Some(carriers[3]), - ] - ); - assert!(model.is_committed_anchor(anchor_vertex)); - assert!(model.is_committed_anchor(regressing_vertex)); - assert!( - model.committed_frontier_dominates(model.effective_frontier(anchor_vertex).unwrap()) - ); + assert!(matches!( + model.record_committed_anchor(regressing_vertex), + Err(CertifiedProjectionError::FrontierRegressesCommitted { authority: 0, .. }) + )); assert!( - model + !model .committed_frontier_dominates(model.effective_frontier(regressing_vertex).unwrap()) ); } @@ -2237,348 +1419,6 @@ mod tests { ); } - #[test] - fn c1_waits_for_an_exact_vote_quorum_and_returns_only_its_witness() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); - let slot = model.leader_slot(1); - let leader = round_one_vertices[slot.author as usize]; - let frontier = round_one_carriers - .iter() - .copied() - .map(Some) - .collect::>(); - - let choices = [ - LeaderChoiceV1::Vote { leader }, - LeaderChoiceV1::Vote { leader }, - LeaderChoiceV1::Vote { leader }, - LeaderChoiceV1::NoVote { - leader_author: slot.author, - leader_round: slot.round, - }, - ]; - let mut projected = BTreeMap::new(); - for author in [0, 1, 3] { - let parents = if author == 3 { - round_one_vertices - .iter() - .copied() - .filter(|parent| parent.author() != slot.author) - .collect() - } else { - round_one_vertices.clone() - }; - let carrier = candidate( - &model.committee, - author, - 2, - &round_one_carriers, - Some(ConsensusVertexV1::new( - 2, - parents, - frontier.clone(), - choices[author as usize], - )), - 0xD0 + author as u8, - ); - let carrier = clean(&mut model, carrier); - projected.insert(author, model.try_project(carrier).unwrap()); - } - assert_eq!(model.c1_strong_parent_witness(3, &[]).unwrap(), None); - - let author = 2; - let carrier = candidate( - &model.committee, - author, - 2, - &round_one_carriers, - Some(ConsensusVertexV1::new( - 2, - round_one_vertices, - frontier, - choices[author as usize], - )), - 0xD0 + author as u8, - ); - let carrier = clean(&mut model, carrier); - projected.insert(author, model.try_project(carrier).unwrap()); - - let witness = model - .c1_strong_parent_witness(3, &[]) - .unwrap() - .expect("the third exact vote completes C1"); - assert_eq!( - witness, - C1StrongParentWitnessV1::Vote { - leader, - parents: [0, 1, 2] - .into_iter() - .map(|author| projected[&author]) - .collect(), - } - ); - let C1StrongParentWitnessV1::Vote { parents, .. } = witness else { - panic!("the exact vote quorum must return a vote witness"); - }; - assert!(!parents.iter().any(|parent| parent.author() == 3)); - } - - #[test] - fn c1_exact_vote_witness_is_not_hidden_by_a_smaller_byzantine_equivocation() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (_, round_one_vertices) = first_consensus_round(&mut model); - let slot = model.leader_slot(1); - let leader = round_one_vertices[slot.author as usize]; - let no_vote = LeaderChoiceV1::NoVote { - leader_author: slot.author, - leader_round: slot.round, - }; - - let byzantine_no_vote = consensus_reference(0, 2, 0x10); - let byzantine_vote = consensus_reference(0, 2, 0xF0); - assert!(byzantine_no_vote < byzantine_vote); - model.inject_projected_for_test(byzantine_no_vote, Vec::new(), no_vote); - model.inject_projected_for_test( - byzantine_vote, - Vec::new(), - LeaderChoiceV1::Vote { leader }, - ); - - let voter_one = consensus_reference(1, 2, 0x21); - let voter_two = consensus_reference(2, 2, 0x22); - let non_voter = consensus_reference(3, 2, 0x23); - for voter in [voter_one, voter_two] { - model.inject_projected_for_test(voter, Vec::new(), LeaderChoiceV1::Vote { leader }); - } - model.inject_projected_for_test(non_voter, Vec::new(), no_vote); - - assert_eq!( - model.c1_strong_parent_witness(3, &[]).unwrap(), - Some(C1StrongParentWitnessV1::Vote { - leader, - parents: vec![byzantine_vote, voter_one, voter_two], - }) - ); - } - - #[test] - fn joined_parent_frontier_omits_unrelated_fresh_closed_tips() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (carriers, vertices) = first_consensus_round(&mut model); - assert_eq!( - model.closed_frontier(), - carriers.iter().copied().map(Some).collect::>() - ); - - let joined = model.joined_strong_parent_frontier(&vertices[..3]).unwrap(); - assert_eq!( - joined, - vec![ - Some(carriers[0]), - Some(carriers[1]), - Some(carriers[2]), - None - ] - ); - } - - #[test] - fn frontier_fresh_quorum_preserves_weighted_stake_and_parent_budget() { - let committee = Committee::new_test(vec![4, 3, 2, 1, 1]); - let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let previous = previous_carriers(&committee, 0); - let mut vertices = Vec::new(); - for author in committee.authorities() { - let carrier = candidate(&committee, author, 1, &previous, None, 0x70 + author as u8); - let carrier_reference = carrier.reference(); - model.stage_carrier(carrier).unwrap(); - let vertex = ConsensusVertexReference::new(carrier_reference, 1); - model.inject_projected_for_test( - vertex, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }, - ); - model.vertices.get_mut(&vertex).unwrap().effective_frontier[author as usize] = - Some(carrier_reference); - vertices.push(vertex); - } - - // The previous lexicographic proof used authors 0, 1, and 2; adding - // required author 4 gave a four-parent hard budget. - let selected = model - .frontier_fresh_quorum(vertices.iter().copied(), &[vertices[4]]) - .unwrap() - .unwrap(); - let selected_stake = selected - .iter() - .map(|reference| committee.get_stake(reference.author()).unwrap()) - .sum::(); - assert!(selected_stake >= committee.quorum_threshold()); - assert!(selected.contains(&vertices[4])); - assert!(selected.len() <= 4); - assert_eq!( - selected - .iter() - .map(|reference| reference.author()) - .collect::>() - .len(), - selected.len(), - "one equivocating author must never contribute stake twice" - ); - } - - fn simulated_parent_frontier_inclusion_lags(frontier_fresh: bool) -> Vec> { - const N: usize = 10; - const ROUNDS: RoundNumber = 30; - - let committee = Committee::new_test(vec![1; N]); - let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let mut previous = previous_carriers(&committee, 0); - let mut vertices_by_round = vec![Vec::new(); ROUNDS as usize + 1]; - - for round in 1..=ROUNDS { - let mut carriers = Vec::with_capacity(N); - for author in committee.authorities() { - let marker = ((round as usize * 17 + author as usize) % 251) as u8; - let carrier = candidate(&committee, author, round, &previous, None, marker); - let reference = carrier.reference(); - model.stage_carrier(carrier).unwrap(); - carriers.push(reference); - } - - let mut round_vertices = Vec::with_capacity(N); - for author in committee.authorities() { - let mut effective_frontier = vec![None; N]; - if round > 1 { - let parent_values = &vertices_by_round[round as usize - 1]; - let leader = parent_values[committee.elect_leader(round - 1) as usize]; - let own = parent_values[author as usize]; - let required = [own, leader]; - let strong_parents = if frontier_fresh { - model - .frontier_fresh_quorum(parent_values.iter().copied(), &required) - .unwrap() - .unwrap() - } else { - let mut selected = BTreeMap::new(); - if round >= 3 { - let mut proof_stake = 0; - for parent in parent_values { - selected.insert(parent.author(), *parent); - proof_stake += committee.get_stake(parent.author()).unwrap(); - if proof_stake >= committee.quorum_threshold() { - break; - } - } - for parent in required { - selected.insert(parent.author(), parent); - } - } else { - for parent in required { - selected.insert(parent.author(), parent); - } - let mut stake = selected - .keys() - .map(|authority| committee.get_stake(*authority).unwrap()) - .sum::(); - for parent in parent_values { - if stake >= committee.quorum_threshold() { - break; - } - if selected.insert(parent.author(), *parent).is_none() { - stake += committee.get_stake(parent.author()).unwrap(); - } - } - } - selected.into_values().collect() - }; - effective_frontier = model - .joined_strong_parent_frontier(&strong_parents) - .unwrap(); - } - effective_frontier[author as usize] = Some(carriers[author as usize]); - let vertex = ConsensusVertexReference::new(carriers[author as usize], round); - model.inject_projected_for_test( - vertex, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: committee.elect_leader(round.saturating_sub(1)), - leader_round: round.saturating_sub(1), - }, - ); - model.vertices.get_mut(&vertex).unwrap().effective_frontier = effective_frontier; - round_vertices.push(vertex); - } - vertices_by_round[round as usize] = round_vertices; - previous = carriers; - } - - let mut lags = vec![Vec::new(); N]; - // Leave a complete leader rotation at the tail so the intentionally - // biased baseline also has time to include every high-index author. - for application_round in 3..=ROUNDS - 12 { - for author in committee.authorities() { - let first_anchor = (application_round..=ROUNDS - 2) - .find(|anchor_round| { - let leader = vertices_by_round[*anchor_round as usize] - [committee.elect_leader(*anchor_round) as usize]; - model.effective_frontier(leader).unwrap()[author as usize] - .is_some_and(|tip| tip.round >= application_round) - }) - .expect("every healthy application prefix must reach a committed leader"); - lags[author as usize].push(first_anchor - application_round); - } - } - lags - } - - #[test] - fn frontier_fresh_quorums_remove_lexicographic_author_inclusion_tail() { - let lexicographic = simulated_parent_frontier_inclusion_lags(false); - let low_author_average = lexicographic[..7] - .iter() - .flatten() - .copied() - .map(u64::from) - .sum::() as f64 - / lexicographic[..7].iter().map(Vec::len).sum::() as f64; - let high_author_average = lexicographic[7..] - .iter() - .flatten() - .copied() - .map(u64::from) - .sum::() as f64 - / lexicographic[7..].iter().map(Vec::len).sum::() as f64; - assert!( - high_author_average > low_author_average + 2.0, - "the regression fixture must expose the former 0..6/7..9 tail" - ); - - let fresh = simulated_parent_frontier_inclusion_lags(true); - let maximum_lag = fresh.iter().flatten().copied().max().unwrap(); - assert!( - maximum_lag <= 2, - "a healthy n=10 frontier should enter a leader within two logical rounds, got {fresh:?}" - ); - let per_author_average = fresh - .iter() - .map(|lags| lags.iter().copied().map(u64::from).sum::() as f64 / lags.len() as f64) - .collect::>(); - let minimum = per_author_average.iter().copied().reduce(f64::min).unwrap(); - let maximum = per_author_average.iter().copied().reduce(f64::max).unwrap(); - assert!( - maximum - minimum < 1.0, - "author skew remains: {per_author_average:?}" - ); - } - #[test] fn direct_skip_uses_clean_projected_explicit_negative_choices() { let committee = Committee::new_test(vec![1; 4]); @@ -2606,7 +1446,7 @@ mod tests { leader_round: slot.round, }; let voter_choices = vec![no_vote, LeaderChoiceV1::Vote { leader }, no_vote, no_vote]; - let (_, voters) = project_complete_round( + project_complete_round( &mut model, 2, &round_one_carriers, @@ -2619,13 +1459,6 @@ mod tests { model.direct_decision(slot).unwrap(), ProjectionDecisionV1::DirectSkip { slot } ); - assert_eq!( - model.c1_strong_parent_witness(3, &[]).unwrap(), - Some(C1StrongParentWitnessV1::DirectSkip { - slot, - parents: [voters[0], voters[2], voters[3]].to_vec(), - }) - ); } fn indirect_graph( @@ -2768,11 +1601,12 @@ mod tests { ); let anchor_carrier = clean(&mut model, anchor_carrier); let anchor = model.try_project(anchor_carrier).unwrap(); + model.record_committed_anchor(anchor).unwrap(); (model, slot, leader, anchor) } #[test] - fn later_selected_anchor_drives_indirect_commit_or_skip_before_frontier_application() { + fn later_committed_anchor_drives_indirect_commit_or_skip() { let (commit_model, slot, leader, commit_anchor) = indirect_graph(true); assert_eq!( commit_model.indirect_decision(slot, commit_anchor).unwrap(), @@ -2791,61 +1625,4 @@ mod tests { } ); } - - #[test] - fn already_dominated_anchor_identity_remains_usable_for_indirect_decision() { - let (mut model, slot, leader, anchor) = indirect_graph(true); - let anchor_frontier = model.effective_frontier(anchor).unwrap().to_vec(); - let anchor_previous = model - .carriers - .get(&anchor.carrier()) - .unwrap() - .candidate - .header() - .own_prev(); - let previous = model - .committee - .authorities() - .map(|authority| { - if authority == anchor.author() { - anchor_previous - } else { - model.closed_tip(authority).unwrap() - } - }) - .collect::>(); - let extension = candidate(&model.committee, 1, 4, &previous, None, 0xB0); - let extension = clean(&mut model, extension); - - let dominating = consensus_reference(1, 5, 0xB1); - model.inject_projected_for_test( - dominating, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: model.committee.elect_leader(4), - leader_round: 4, - }, - ); - let mut dominating_frontier = anchor_frontier; - dominating_frontier[1] = Some(extension); - model - .vertices - .get_mut(&dominating) - .unwrap() - .effective_frontier - .clone_from(&dominating_frontier); - assert_eq!( - model.record_committed_anchor(dominating).unwrap(), - dominating_frontier - ); - assert_eq!( - model.record_committed_anchor(anchor).unwrap(), - dominating_frontier - ); - assert!(model.is_committed_anchor(anchor)); - assert_eq!( - model.indirect_decision(slot, anchor).unwrap(), - ProjectionDecisionV1::IndirectCommit { leader, anchor } - ); - } } diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 8cb806d7..5caa8c7a 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -10,7 +10,7 @@ //! only after that complete batch has reached durable storage. use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet}, error::Error, fmt, path::Path, @@ -25,27 +25,22 @@ use crate::{ LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, carrier_genesis_reference, journal::{IngressProvenanceV1, JournalErrorV1, JournalEventV1, WriteAheadJournalV1}, - model::{ - DeliveryPromiseBasisV1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, ModelError, - ModelInputRecord, ModelTraceEvent, RbcDagModel, - }, + model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, projection::{ - C1StrongParentWitnessV1, CertifiedProjectionError, CertifiedProjectionModel, - LeaderSlotV1, ProjectionDecisionV1, PromisedProjectionModel, + CertifiedProjectionError, CertifiedProjectionModel, LeaderSlotV1, ProjectionDecisionV1, }, storage::{ MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, ShadowWalSummaryV1, ShadowWalSyncPolicyV1, ShadowWalV1, }, }, - store::RbcDagFrontierReceipt, types::{ AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, Stake, TimestampNs, }, }; -const RAW_RECORD_MAGIC: &[u8; 4] = b"SRD5"; +const RAW_RECORD_MAGIC: &[u8; 4] = b"SRD3"; const RAW_RECORD_VERSION_V1: u8 = 1; const RAW_RECORD_HEADER_SIZE: usize = 80; @@ -67,19 +62,14 @@ const TRACE_DELIVERY_LOCKED: u8 = 0x05; const TRACE_EFFECT: u8 = 0x06; const TRACE_CONSENSUS_SLOT_LOCKED: u8 = 0x07; const TRACE_LEADER_CHOICE_LOCKED: u8 = 0x08; -const TRACE_DELIVERY_PROMISE_LOCKED: u8 = 0x09; const EFFECT_NEED_CARRIER: u8 = 0x00; const EFFECT_DELIVERED: u8 = 0x01; const EFFECT_PREFIX_ADVANCED: u8 = 0x02; const EFFECT_CARRIER_ROUND_ADVANCED: u8 = 0x03; -const EFFECT_DELIVERY_PROMISED: u8 = 0x04; const PHASE_ECHO: u8 = 0x00; const PHASE_READY: u8 = 0x01; -// Phase codes are append-only because they are persisted in the shadow WAL. -const PHASE_VOTE: u8 = 0x02; -const PHASE_ACK: u8 = 0x03; const PROVENANCE_DIRECT: u8 = 0x00; const PROVENANCE_RELAYED: u8 = 0x01; @@ -89,10 +79,6 @@ const PROVENANCE_RELAYED: u8 = 0x01; /// protocol must derive pruning from a certified/committed watermark instead. /// Exact RBC recovery requests are exempt from this prototype guard. const SHADOW_BENCHMARK_UNSOLICITED_RETENTION_WINDOW_ROUNDS_V1: RoundNumber = 64; -/// A restart may replay only a bounded committed-frontier suffix into Core. -/// Larger gaps require a future checkpoint-transfer protocol rather than an -/// unbounded startup event burst. -pub(crate) const MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1: usize = 64; /// Local authentication material owned by exactly one shadow core. /// @@ -155,10 +141,6 @@ pub(crate) enum ShadowIngressDispositionV1 { Authenticated, CandidateRetained, IgnoredDuplicateConflictOrStale, - /// An unsolicited carrier was beyond the bounded future-retention - /// window. It was discarded before authentication and without WAL/model - /// mutation; exact current-round synchronization remains available. - IgnoredFutureOutsideBuffer, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -241,77 +223,16 @@ pub(crate) enum ShadowDeliveryComparisonV1 { }, } -/// Cheap, non-consensus physical-round observation for one first-committed -/// application. Unlike the discarded logical-ancestry diagnostic, this is -/// computed once at output and adds no work to projection. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct CommittedApplicationDiagnosticV1 { - pub(crate) physical_carrier_round_delta: i64, -} - -impl CommittedApplicationDiagnosticV1 { - fn new(anchor: ConsensusVertexReference, enclosing_carrier: BlockReference) -> Self { - Self { - physical_carrier_round_delta: i64::from(anchor.carrier().round) - - i64::from(enclosing_carrier.round), - } - } -} - /// Deterministic application output unlocked by one newly committed clean /// projected anchor. Carrier references remain available for audit while the /// application headers are already deduplicated and sorted by their carrier /// position in the exact committed frontier delta. #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CommittedFrontierDeltaV1 { - /// Monotone one-based output position, independent of the possibly - /// regressing logical round of `anchor`. - pub(crate) output_sequence: RoundNumber, pub(crate) anchor: ConsensusVertexReference, pub(crate) frontier: Vec>, pub(crate) carriers: Vec, pub(crate) applications: Vec, - /// Diagnostic-only records in one-to-one order with `applications`. - pub(crate) application_diagnostics: Vec, -} - -/// Runtime-only handoff between Core's compact durable receipt and the -/// authoritative carrier WAL. Application references are reconstructed from -/// the CommitData atomically stored under `receipt.carrier_anchor`; a missing -/// CommitData value therefore denotes an exact control-only frontier. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct RbcDagFrontierRecoveryCursorV1 { - pub(crate) receipt: RbcDagFrontierReceipt, - pub(crate) application_references: Vec, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ProjectionHolReasonV1 { - InsufficientLookahead, - DirectEvidencePending, - AwaitingIndirectAnchor, - Ready, -} - -impl ProjectionHolReasonV1 { - pub(crate) const fn metric_label(self) -> &'static str { - match self { - Self::InsufficientLookahead => "insufficient_lookahead", - Self::DirectEvidencePending => "direct_evidence_pending", - Self::AwaitingIndirectAnchor => "awaiting_indirect_anchor", - Self::Ready => "ready", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct ProjectionRuntimeSnapshotV1 { - pub(crate) pending_candidates: usize, - pub(crate) highest_projected_round: RoundNumber, - pub(crate) next_undecided_round: RoundNumber, - pub(crate) next_undecided_projected_stake: Stake, - pub(crate) last_committed_round: RoundNumber, - pub(crate) hol_reason: ProjectionHolReasonV1, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -319,7 +240,6 @@ pub(crate) struct ShadowOpenReportV1 { replayed_batches: u64, discarded_tail_bytes: u64, recovery_effects: Vec, - recovered_committed_frontiers: Vec, } impl ShadowOpenReportV1 { @@ -336,24 +256,6 @@ impl ShadowOpenReportV1 { pub(crate) fn recovery_effects(&self) -> &[ModelEffect] { &self.recovery_effects } - - /// Strictly newer authoritative WAL output that Core has not durably - /// acknowledged yet. Observational opens always return an empty suffix. - pub(crate) fn recovered_committed_frontiers(&self) -> &[CommittedFrontierDeltaV1] { - &self.recovered_committed_frontiers - } - - #[cfg(test)] - pub(crate) fn with_recovered_committed_frontiers_for_test( - recovered_committed_frontiers: Vec, - ) -> Self { - Self { - replayed_batches: 1, - discarded_tail_bytes: 0, - recovery_effects: Vec::new(), - recovered_committed_frontiers, - } - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -374,7 +276,6 @@ pub(crate) enum ShadowCodecErrorV1 { InvalidProvenance(u8), InvalidPhase(u8), InvalidLeaderChoice(u8), - InvalidDeliveryPromiseBasis(u8), InvalidTrace(u8), InvalidEffect(u8), NonCanonicalHolders, @@ -415,43 +316,6 @@ pub(crate) enum ShadowErrorV1 { batch_sequence: u64, reason: &'static str, }, - FrontierRecoveryWatermarkLength { - expected: usize, - actual: usize, - }, - FrontierOutputSequenceOverflow(u64), - FrontierRecoverySequence { - expected_sequence: RoundNumber, - actual_sequence: RoundNumber, - }, - FrontierRecoveryCursorAhead { - durable_sequence: RoundNumber, - actor_sequence: RoundNumber, - }, - FrontierRecoveryCursorMissing(RoundNumber), - FrontierRecoveryAnchorConflict { - consensus_round: RoundNumber, - durable: BlockReference, - actor: BlockReference, - }, - FrontierRecoveryApplicationsConflict { - consensus_round: RoundNumber, - durable: Vec, - actor: Vec, - }, - FrontierRecoveryWatermarksConflict { - consensus_round: RoundNumber, - durable: Vec, - actor: Vec, - }, - FrontierRecoveryApplicationAuthority { - application: BlockReference, - committee_size: usize, - }, - FrontierRecoverySuffixLimit { - limit: usize, - actual: usize, - }, UnrequestedRecovery(BlockReference), SlotCandidateLimit { author: AuthorityIndex, @@ -506,67 +370,6 @@ impl fmt::Display for ShadowErrorV1 { formatter, "shadow replay policy violation in WAL batch {batch_sequence}: {reason}" ), - Self::FrontierRecoveryWatermarkLength { expected, actual } => write!( - formatter, - "RBC-DAG frontier recovery cursor watermark length mismatch: expected {expected}, got {actual}" - ), - Self::FrontierOutputSequenceOverflow(sequence) => write!( - formatter, - "RBC-DAG frontier output sequence {sequence} exceeds the u32 receipt encoding" - ), - Self::FrontierRecoverySequence { - expected_sequence, - actual_sequence, - } => write!( - formatter, - "RBC-DAG frontier recovery output sequence mismatch: expected {expected_sequence}, got {actual_sequence}" - ), - Self::FrontierRecoveryCursorAhead { - durable_sequence, - actor_sequence, - } => write!( - formatter, - "RBC-DAG frontier recovery cursor sequence {durable_sequence} is ahead of actor WAL sequence {actor_sequence}" - ), - Self::FrontierRecoveryCursorMissing(sequence) => write!( - formatter, - "RBC-DAG frontier recovery cursor sequence {sequence} is missing from the actor WAL" - ), - Self::FrontierRecoveryAnchorConflict { - consensus_round, - durable, - actor, - } => write!( - formatter, - "RBC-DAG frontier recovery anchor conflict at consensus round {consensus_round}: durable {durable}, actor {actor}" - ), - Self::FrontierRecoveryApplicationsConflict { - consensus_round, - durable, - actor, - } => write!( - formatter, - "RBC-DAG frontier recovery application conflict at consensus round {consensus_round}: durable {durable:?}, actor {actor:?}" - ), - Self::FrontierRecoveryWatermarksConflict { - consensus_round, - durable, - actor, - } => write!( - formatter, - "RBC-DAG frontier recovery watermark conflict at consensus round {consensus_round}: durable {durable:?}, actor {actor:?}" - ), - Self::FrontierRecoveryApplicationAuthority { - application, - committee_size, - } => write!( - formatter, - "RBC-DAG frontier recovery application {application} is outside committee size {committee_size}" - ), - Self::FrontierRecoverySuffixLimit { limit, actual } => write!( - formatter, - "RBC-DAG frontier recovery suffix exceeds the bound {limit}: got {actual}" - ), Self::UnrequestedRecovery(reference) => { write!(formatter, "unrequested shadow recovery for {reference}") } @@ -714,30 +517,17 @@ pub(crate) struct StarfishRbcDagShadowV1 { wal: ShadowWalV1, candidates: BTreeMap, application_carriers: BTreeMap>, - /// Every authoritative delivery, including the optimistic ECHO path. delivered: BTreeSet, - /// The slower fallback values that additionally reached `Q` READYs. - certified_delivered: BTreeSet, authenticated_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, requested_recoveries: BTreeMap>, - promised_projection: PromisedProjectionModel, - pending_promised_references: BTreeSet, - pending_promised_projection_candidates: BTreeSet, - promised_projection_rejected: BTreeMap, projection: CertifiedProjectionModel, pending_projection_candidates: BTreeSet, projection_rejected: BTreeMap, projected_decisions: BTreeSet, - /// Exact derived index for the append-only decision history. Consensus - /// drive asks this on every newly decidable round; scanning all prior - /// decisions made a long run quadratic despite the slot being unique. - projected_decision_slots: BTreeSet, included_applications: BTreeSet, highest_projected_consensus_round: RoundNumber, - highest_committed_consensus_round: RoundNumber, - committed_output_count: u64, next_undecided_consensus_round: RoundNumber, next_local_consensus_round: RoundNumber, pending_projected_vertices: Vec, @@ -746,15 +536,6 @@ pub(crate) struct StarfishRbcDagShadowV1 { poisoned: bool, } -enum ShadowFrontierRecoveryPolicyV1 { - /// Comparison/mirror mode reconstructs state but never republishes - /// historical authoritative outputs. - Observational, - /// Embedded-authority mode reconciles Core's exact durable cursor and - /// republishes only the bounded, strictly newer actor-WAL suffix. - Authoritative(Option), -} - impl StarfishRbcDagShadowV1 { #[cfg(test)] pub(crate) fn open( @@ -781,54 +562,8 @@ impl StarfishRbcDagShadowV1 { context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, wal_sync_policy: ShadowWalSyncPolicyV1, - ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { - Self::open_with_frontier_recovery_policy( - path, - committee, - own_authority, - context, - authorizer, - wal_sync_policy, - ShadowFrontierRecoveryPolicyV1::Observational, - ) - } - - /// Open the embedded-authority runtime and reconcile the carrier WAL's - /// committed output against Core's exact durable cursor. The report owns - /// the strictly newer bounded suffix so startup can publish it before its - /// readiness barrier. - pub(crate) fn open_authoritative_with_wal_sync_policy( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - wal_sync_policy: ShadowWalSyncPolicyV1, - recovery_cursor: Option, - ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { - Self::open_with_frontier_recovery_policy( - path, - committee, - own_authority, - context, - authorizer, - wal_sync_policy, - ShadowFrontierRecoveryPolicyV1::Authoritative(recovery_cursor), - ) - } - - #[allow(clippy::too_many_arguments)] - fn open_with_frontier_recovery_policy( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - wal_sync_policy: ShadowWalSyncPolicyV1, - frontier_recovery_policy: ShadowFrontierRecoveryPolicyV1, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { validate_configuration(&committee, own_authority, context, &authorizer)?; - let committee_size = committee.committee().len(); let namespace = ShadowWalNamespaceV1::new(context, own_authority); let (wal, recovery) = ShadowWalV1::open_with_sync_policy(path, namespace, wal_sync_policy)?; let replayed_batches = recovery.batch_count(); @@ -836,8 +571,6 @@ impl StarfishRbcDagShadowV1 { let mut model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; model.enable_intrinsic_empty_data_availability(); - let promised_projection = - PromisedProjectionModel::from_committee_context(committee.clone()); let projection = CertifiedProjectionModel::from_committee_context(committee.clone()); let journal = WriteAheadJournalV1::new(context, own_authority); let mut core = Self { @@ -851,24 +584,16 @@ impl StarfishRbcDagShadowV1 { candidates: BTreeMap::new(), application_carriers: BTreeMap::new(), delivered: BTreeSet::new(), - certified_delivered: BTreeSet::new(), authenticated_slots: BTreeMap::new(), ordinarily_retained_slots: BTreeMap::new(), slot_candidates: BTreeMap::new(), requested_recoveries: BTreeMap::new(), - promised_projection, - pending_promised_references: BTreeSet::new(), - pending_promised_projection_candidates: BTreeSet::new(), - promised_projection_rejected: BTreeMap::new(), projection, pending_projection_candidates: BTreeSet::new(), projection_rejected: BTreeMap::new(), projected_decisions: BTreeSet::new(), - projected_decision_slots: BTreeSet::new(), included_applications: BTreeSet::new(), highest_projected_consensus_round: 0, - highest_committed_consensus_round: 0, - committed_output_count: 0, next_undecided_consensus_round: 1, next_local_consensus_round: 1, pending_projected_vertices: Vec::new(), @@ -881,22 +606,11 @@ impl StarfishRbcDagShadowV1 { let input = core.decode_batch(batch.records())?; core.apply_replayed(input, batch.records(), batch.sequence())?; } - // Historical decisions are reconstructed to validate replay, but only - // an authoritative open may republish the exact Core-unacknowledged - // committed-frontier suffix. + // Historical decisions are reconstructed to validate replay, but are + // not re-emitted as fresh runtime observations after restart. core.pending_projection_decisions.clear(); core.pending_projected_vertices.clear(); - let replayed_frontiers = std::mem::take(&mut core.pending_committed_frontiers); - let recovered_committed_frontiers = match frontier_recovery_policy { - ShadowFrontierRecoveryPolicyV1::Observational => Vec::new(), - ShadowFrontierRecoveryPolicyV1::Authoritative(cursor) => { - reconcile_authoritative_frontier_suffix( - replayed_frontiers, - cursor.as_ref(), - committee_size, - )? - } - }; + core.pending_committed_frontiers.clear(); let recovery_effects = core .requested_recoveries .iter() @@ -911,7 +625,6 @@ impl StarfishRbcDagShadowV1 { replayed_batches, discarded_tail_bytes, recovery_effects, - recovered_committed_frontiers, }, )) } @@ -924,14 +637,6 @@ impl StarfishRbcDagShadowV1 { self.model.can_create_carrier() } - /// Whether the open local carrier can name an exact admitted quorum from - /// the immediately preceding physical round. This is a read-only pacing - /// predicate: the reducer still validates the same parent set again when - /// the carrier is durably fixed. - pub(crate) fn local_parent_quorum_ready(&self) -> bool { - self.model.local_parent_set().is_ok() - } - pub(crate) fn pending_phase_backlog_len(&self) -> usize { self.model.pending_phase_backlog_len() } @@ -948,7 +653,6 @@ impl StarfishRbcDagShadowV1 { }) } - #[cfg(test)] pub(crate) fn admitted_reference( &self, authority: AuthorityIndex, @@ -957,17 +661,6 @@ impl StarfishRbcDagShadowV1 { self.model.admitted_reference(authority, round) } - /// Return the exact authenticated value retained for a slot, including a - /// future value that is inside the normal 64-round window but not yet - /// admitted by sequential clock advancement. - pub(crate) fn authenticated_reference( - &self, - authority: AuthorityIndex, - round: RoundNumber, - ) -> Option { - self.authenticated_slots.get(&(authority, round)).copied() - } - pub(crate) fn current_round_admitted_author_count(&self) -> usize { let round = self.local_carrier_round(); self.committee @@ -1019,7 +712,11 @@ impl StarfishRbcDagShadowV1 { reference: BlockReference, ) -> Result, ShadowErrorV1> { self.ensure_live()?; - if self.projection.is_data_available(reference) { + if self + .model + .lifecycle(&reference) + .is_some_and(|lifecycle| lifecycle.data_available) + { return Ok(Vec::new()); } self.apply_durable(ShadowInputV1::DataAvailable(reference)) @@ -1033,7 +730,9 @@ impl StarfishRbcDagShadowV1 { } pub(crate) fn carrier_data_available(&self, reference: BlockReference) -> bool { - self.projection.is_data_available(reference) + self.model + .lifecycle(&reference) + .is_some_and(|lifecycle| lifecycle.data_available) } pub(crate) fn wal_counts(&self) -> (u64, u64) { @@ -1049,23 +748,6 @@ impl StarfishRbcDagShadowV1 { self.model.delivered(authority, round) } - #[cfg(test)] - pub(crate) fn optimistic_promise_count(&self) -> usize { - self.slot_candidates - .values() - .flat_map(BTreeSet::iter) - .filter(|reference| { - self.model.delivery_promise_basis(reference) - == Some(DeliveryPromiseBasisV1::OptimisticEcho) - }) - .count() - } - - #[cfg(test)] - pub(crate) fn certified_delivery_count(&self) -> Result { - Ok(self.certified_delivered.len()) - } - /// Construct, authenticate, durably fix, and expose the next local /// carrier. M3 intentionally uses empty ACKs and no consensus vertex. pub(crate) fn create_local_carrier( @@ -1175,10 +857,6 @@ impl StarfishRbcDagShadowV1 { own_prev: BlockReference, allow_no_vote: bool, ) -> Option { - // `allow_no_vote` authorizes the complete C2/C3 fallback, including a - // late Vote when the scheduled leader exists. The service must bind - // that authorization to this logical consensus slot, not to a global - // physical-carrier heartbeat. let consensus_round = self.next_local_consensus_round(); let (strong_parents, leader_choice) = if consensus_round == 1 { let strong_parents = self @@ -1202,10 +880,7 @@ impl StarfishRbcDagShadowV1 { } else { let parent_round = consensus_round - 1; let mut by_author = BTreeMap::new(); - for parent in self - .promised_projection - .projected_values_at_round(parent_round) - { + for parent in self.projection.projected_values_at_round(parent_round) { by_author.entry(parent.author()).or_insert(parent); } let own_parent = *by_author.get(&self.own_authority)?; @@ -1217,76 +892,20 @@ impl StarfishRbcDagShadowV1 { return None; } let leader_author = self.committee.committee().elect_leader(parent_round); - let leader = by_author.get(&leader_author).copied(); - let (strong_parents, leader_choice) = if consensus_round >= 3 { - let witness = match leader { - Some(leader) => match self - .promised_projection - .c1_strong_parent_witness(consensus_round, &[own_parent, leader]) - { - Ok(witness) => witness, - Err(_) => return None, - }, - None => None, - }; - match (leader, witness) { - (Some(leader), Some(witness)) => { - let witness_parents = match &witness { - C1StrongParentWitnessV1::Vote { - leader: witnessed, - parents, - } => { - debug_assert_eq!( - witnessed.consensus_round(), - consensus_round.saturating_sub(2) - ); - parents - } - C1StrongParentWitnessV1::DirectSkip { slot, parents } => { - debug_assert_eq!(slot.round, consensus_round.saturating_sub(2)); - parents - } - }; - if !witness_parents.contains(&own_parent) - || !witness_parents.contains(&leader) - { - return None; - } - (witness_parents.clone(), LeaderChoiceV1::Vote { leader }) - } - _ if !allow_no_vote => return None, - _ => self.fallback_strong_parents( - &by_author, - own_parent, - leader, - leader_author, - parent_round, - )?, - } - } else { - match leader { - Some(leader) => ( - self.minimal_strong_parent_quorum(&by_author, [own_parent, leader])?, - LeaderChoiceV1::Vote { leader }, - ), - None if allow_no_vote => self.fallback_strong_parents( - &by_author, - own_parent, - None, - leader_author, - parent_round, - )?, - None => return None, - } + let leader_choice = match by_author.get(&leader_author).copied() { + Some(leader) => LeaderChoiceV1::Vote { leader }, + None if allow_no_vote => LeaderChoiceV1::NoVote { + leader_author, + leader_round: parent_round, + }, + None => return None, }; + let strong_parents = by_author.into_values().collect::>(); debug_assert!(strong_parents.contains(&own_parent)); (strong_parents, leader_choice) }; - let mut delivery_frontier = self - .promised_projection - .joined_strong_parent_frontier(&strong_parents) - .ok()?; + let mut delivery_frontier = self.projection.closed_frontier(); let own_entry = (own_prev.round != 0).then_some(own_prev); // The enclosing carrier is not clean yet, so its immediate physical // predecessor may be ahead of today's closed tip. The immutable @@ -1302,104 +921,10 @@ impl StarfishRbcDagShadowV1 { )) } - fn fallback_strong_parents( - &self, - by_author: &BTreeMap, - own_parent: ConsensusVertexReference, - leader: Option, - leader_author: AuthorityIndex, - leader_round: RoundNumber, - ) -> Option<(Vec, LeaderChoiceV1)> { - match leader { - Some(leader) => Some(( - self.minimal_strong_parent_quorum(by_author, [own_parent, leader])?, - LeaderChoiceV1::Vote { leader }, - )), - None => Some(( - self.minimal_strong_parent_quorum(by_author, [own_parent])?, - LeaderChoiceV1::NoVote { - leader_author, - leader_round, - }, - )), - } - } - - fn minimal_strong_parent_quorum( - &self, - by_author: &BTreeMap, - required: [ConsensusVertexReference; N], - ) -> Option> { - self.promised_projection - .frontier_fresh_quorum(by_author.values().copied(), &required) - .ok() - .flatten() - } - - pub(crate) fn next_local_consensus_round(&self) -> RoundNumber { + fn next_local_consensus_round(&self) -> RoundNumber { self.next_local_consensus_round } - pub(crate) fn projected_consensus_stake(&self, round: RoundNumber) -> Stake { - self.promised_projection.projected_stake_at_round(round) - } - - pub(crate) fn has_projected_consensus_quorum(&self, round: RoundNumber) -> bool { - self.projected_consensus_stake(round) >= self.committee.committee().quorum_threshold() - } - - #[cfg(test)] - pub(crate) fn inject_projected_consensus_for_test( - &mut self, - reference: ConsensusVertexReference, - strong_parents: Vec, - leader_choice: LeaderChoiceV1, - ) { - self.promised_projection.inject_projected_for_test( - reference, - strong_parents, - leader_choice, - ); - } - - #[cfg(test)] - pub(crate) fn set_next_local_consensus_round_for_test(&mut self, round: RoundNumber) { - self.next_local_consensus_round = round; - } - - pub(crate) fn projection_runtime_snapshot(&self) -> ProjectionRuntimeSnapshotV1 { - let decidable_round = self.highest_projected_consensus_round.saturating_sub(2); - let hol_reason = if self.next_undecided_consensus_round > decidable_round { - ProjectionHolReasonV1::InsufficientLookahead - } else { - let slot = self - .projection - .leader_slot(self.next_undecided_consensus_round); - match self.projection.direct_decision(slot) { - Err(_) => ProjectionHolReasonV1::DirectEvidencePending, - Ok(ProjectionDecisionV1::Undecided { .. }) => { - ProjectionHolReasonV1::AwaitingIndirectAnchor - } - Ok( - ProjectionDecisionV1::DirectCommit { .. } - | ProjectionDecisionV1::DirectSkip { .. } - | ProjectionDecisionV1::IndirectCommit { .. } - | ProjectionDecisionV1::IndirectSkip { .. }, - ) => ProjectionHolReasonV1::Ready, - } - }; - ProjectionRuntimeSnapshotV1 { - pending_candidates: self.pending_projection_candidates.len(), - highest_projected_round: self.highest_projected_consensus_round, - next_undecided_round: self.next_undecided_consensus_round, - next_undecided_projected_stake: self - .projection - .projected_stake_at_round(self.next_undecided_consensus_round), - last_committed_round: self.highest_committed_consensus_round, - hol_reason, - } - } - /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -1443,26 +968,6 @@ impl StarfishRbcDagShadowV1 { canonical_carrier_wire: &[u8], authentication_sidecar: &[u8], trusted_peer: AuthorityIndex, - ) -> Result { - self.receive_or_retain_from_peer_with_future_window( - canonical_carrier_wire, - authentication_sidecar, - trusted_peer, - EXECUTABLE_MODEL_BUFFER_WINDOW_V1, - ) - } - - /// Normal operation retains the full prototype elasticity window. During - /// exact catch-up the service narrows unsolicited ingress to the admission - /// horizon so replay responses are not starved behind duplicate future - /// work; requested exact-slot synchronization uses the normal window and - /// remains unaffected. - pub(crate) fn receive_or_retain_from_peer_with_future_window( - &mut self, - canonical_carrier_wire: &[u8], - authentication_sidecar: &[u8], - trusted_peer: AuthorityIndex, - future_window: RoundNumber, ) -> Result { self.ensure_live()?; if !self.committee.committee().known_authority(trusted_peer) { @@ -1470,21 +975,6 @@ impl StarfishRbcDagShadowV1 { } let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); - // A healthy quorum may run ahead of a temporarily descheduled peer, - // but arbitrary unsolicited future traffic must not consume MAC or - // signature verification and durable reducer capacity. Exact sync - // requests recover the receiver's current slot one round at a time. - if candidate.reference().round - > self - .model - .local_carrier_round() - .saturating_add(future_window.min(EXECUTABLE_MODEL_BUFFER_WINDOW_V1)) - { - return Ok(ShadowIngressOutcomeV1::new( - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer, - Vec::new(), - )); - } // Once a slot has a durably authenticated value, unsolicited replays // and conflicts cannot change the shadow state. Reject before public // signature/ML-DSA verification to keep this idempotence cheap. @@ -1778,7 +1268,7 @@ impl StarfishRbcDagShadowV1 { Ok(self.wal.shutdown()?) } - fn drive_certified_projection(&mut self) -> Result<(), ShadowErrorV1> { + fn drive_certified_projection(&mut self) { loop { let mut advanced = false; let candidates = self @@ -1808,204 +1298,125 @@ impl StarfishRbcDagShadowV1 { } } - self.drive_ordered_committer(self.highest_projected_consensus_round) + self.drive_ordered_committer(self.highest_projected_consensus_round); } - /// Advance the planning-only projection without emitting vertices, - /// decisions, anchors, or committed frontiers. A rejected promised value - /// is isolated here; certified projection continues independently. - fn drive_promised_projection(&mut self) { + fn drive_ordered_committer(&mut self, highest_round: RoundNumber) { + let decidable_round = highest_round.saturating_sub(2); loop { - let mut advanced = false; - let candidates = self - .pending_promised_projection_candidates - .iter() - .copied() - .collect::>(); - for reference in candidates { - match self.promised_projection.try_project(reference) { - Ok(_) => { - self.pending_promised_projection_candidates - .remove(&reference); - advanced = true; - } - Err(error) if projection_error_is_pending(&error) => {} - Err(error) => { - self.pending_promised_projection_candidates - .remove(&reference); - self.promised_projection_rejected.insert(reference, error); - } - } + while self.next_undecided_consensus_round <= decidable_round + && self.has_projection_decision( + self.projection + .leader_slot(self.next_undecided_consensus_round), + ) + { + self.next_undecided_consensus_round = + self.next_undecided_consensus_round.saturating_add(1); } - if !advanced { - break; + if self.next_undecided_consensus_round > decidable_round { + return; } - } - } - - /// Resolve durable promise effects only once exact canonical content is - /// locally staged. Missing content remains an explicit replay-derived - /// pending reference until authenticated ingress or exact recovery stores - /// it; no placeholder can enter either projection plane or the delivered - /// application set. - fn activate_promised_references(&mut self) { - let available = self - .pending_promised_references - .iter() - .copied() - .filter(|reference| self.promised_projection.carrier_is_stored(*reference)) - .collect::>(); - for reference in available { - self.promised_projection - .mark_promised(reference) - .expect("a stored model promise must be valid in the promised plane"); - self.projection - .mark_delivered(reference) - .expect("a stored optimistic delivery must be valid in the certified plane"); - self.delivered.insert(reference); - self.requested_recoveries.remove(&reference); - self.pending_promised_references.remove(&reference); - } - } - - fn drive_ordered_committer(&mut self, highest_round: RoundNumber) -> Result<(), ShadowErrorV1> { - let decidable_round = highest_round.saturating_sub(2); - if self.next_undecided_consensus_round > decidable_round { - return Ok(()); - } - - // Plan from newest to oldest, as the universal Starfish committer - // does. An indirect decision may use only the first committed leader - // in this already-decided suffix; a still-undecided intervening slot - // is a hard barrier. This makes anchor selection independent of which - // later direct certificate happened to arrive first locally. - let mut planned = VecDeque::new(); - for round in (self.next_undecided_consensus_round..=decidable_round).rev() { + let round = self.next_undecided_consensus_round; let slot = self.projection.leader_slot(round); - let direct = self.projection.direct_decision(slot)?; - let decision = match direct { + let Ok(decision) = self.projection.direct_decision(slot) else { + return; + }; + match decision { + ProjectionDecisionV1::DirectCommit { leader } => { + self.commit_projected_anchor(leader); + self.record_projection_decision(decision); + self.next_undecided_consensus_round = round.saturating_add(1); + } + ProjectionDecisionV1::DirectSkip { .. } => { + self.record_projection_decision(decision); + self.next_undecided_consensus_round = round.saturating_add(1); + } ProjectionDecisionV1::Undecided { .. } => { - let minimum_anchor_round = round.saturating_add(3); - let mut anchor = None; - for later in planned.iter().filter(|later| { - projection_decision_slot(**later).round >= minimum_anchor_round - }) { - match later { - ProjectionDecisionV1::DirectCommit { leader } - | ProjectionDecisionV1::IndirectCommit { leader, .. } => { - anchor = Some(*leader); - break; + let first_anchor_round = round.saturating_add(3); + let later_anchor = + (first_anchor_round..=decidable_round).find_map(|candidate| { + let candidate_slot = self.projection.leader_slot(candidate); + match self.projection.direct_decision(candidate_slot).ok()? { + ProjectionDecisionV1::DirectCommit { leader } => Some(leader), + ProjectionDecisionV1::DirectSkip { .. } + | ProjectionDecisionV1::IndirectCommit { .. } + | ProjectionDecisionV1::IndirectSkip { .. } + | ProjectionDecisionV1::Undecided { .. } => None, } - ProjectionDecisionV1::DirectSkip { .. } - | ProjectionDecisionV1::IndirectSkip { .. } => {} - ProjectionDecisionV1::Undecided { .. } => break, - } - } - let Some(anchor) = anchor else { - planned.push_front(direct); - continue; + }); + let Some(anchor) = later_anchor else { + return; }; - self.projection.indirect_decision(slot, anchor)? + self.commit_projected_anchor(anchor); + let indirect = self + .projection + .indirect_decision(slot, anchor) + .expect("a clean committed later anchor must decide the older slot"); + self.record_projection_decision(indirect); + self.record_projection_decision(ProjectionDecisionV1::DirectCommit { + leader: anchor, + }); + self.next_undecided_consensus_round = round.saturating_add(1); } - ProjectionDecisionV1::DirectCommit { .. } - | ProjectionDecisionV1::DirectSkip { .. } => direct, ProjectionDecisionV1::IndirectCommit { .. } | ProjectionDecisionV1::IndirectSkip { .. } => { unreachable!("direct decision returned an indirect result") } - }; - planned.push_front(decision); - } - - // Emit only the longest finalized prefix. Every committed leader, - // including an indirectly committed one, contributes its frontier at - // its exact position in that agreed leader order. Later deciding - // anchors are therefore never applied ahead of older leaders. - for decision in planned { - if matches!(decision, ProjectionDecisionV1::Undecided { .. }) { - break; - } - match decision { - ProjectionDecisionV1::DirectCommit { leader } - | ProjectionDecisionV1::IndirectCommit { leader, .. } => { - self.commit_projected_anchor(leader)?; - } - ProjectionDecisionV1::DirectSkip { .. } - | ProjectionDecisionV1::IndirectSkip { .. } => {} - ProjectionDecisionV1::Undecided { .. } => unreachable!("handled above"), } - let round = projection_decision_slot(decision).round; - self.record_projection_decision(decision); - self.next_undecided_consensus_round = round.saturating_add(1); } - Ok(()) + } + + fn has_projection_decision(&self, slot: LeaderSlotV1) -> bool { + self.projected_decisions + .iter() + .any(|decision| projection_decision_slot(*decision) == slot) } fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { if self.projected_decisions.insert(decision) { - let slot = projection_decision_slot(decision); - assert!( - self.projected_decision_slots.insert(slot), - "one logical leader slot cannot retain conflicting projection decisions" - ); self.pending_projection_decisions.push(decision); } } - fn commit_projected_anchor( - &mut self, - anchor: ConsensusVertexReference, - ) -> Result<(), ShadowErrorV1> { + fn commit_projected_anchor(&mut self, anchor: ConsensusVertexReference) { if self.projection.is_committed_anchor(anchor) { - return Ok(()); + return; } - let next_count = self - .committed_output_count - .checked_add(1) - .ok_or(ShadowErrorV1::FrontierOutputSequenceOverflow(u64::MAX))?; - let output_sequence = RoundNumber::try_from(next_count) - .map_err(|_| ShadowErrorV1::FrontierOutputSequenceOverflow(next_count))?; let frontier = self .projection - .record_committed_anchor(anchor) - .unwrap_or_else(|error| { - panic!("committed clean anchor frontiers must have an exact-prefix join: {error}") - }); - self.highest_committed_consensus_round = self - .highest_committed_consensus_round - .max(anchor.consensus_round()); + .effective_frontier(anchor) + .expect("a committed anchor must be clean and projected") + .to_vec(); + if let Err(error) = self.projection.record_committed_anchor(anchor) { + if self.projection.committed_frontier_dominates(&frontier) { + // The leader is logically committed, but a later anchor used + // for an indirect decision has already output its complete + // carrier prefix. Re-emitting it would regress the frontier. + return; + } + panic!("ordered clean anchors must be comparable exact frontiers: {error}"); + } let carriers = self .model .apply_frontier(&frontier) .expect("projection and reducer closed prefixes must agree"); - let mut applications = Vec::new(); - let mut application_diagnostics = Vec::new(); - for carrier in &carriers { - let Some(application) = self - .candidates - .get(carrier) - .and_then(|candidate| candidate.header().application_header()) - .cloned() - else { - continue; - }; - if !self.included_applications.insert(application.reference()) { - continue; - } - applications.push(application); - application_diagnostics.push(CommittedApplicationDiagnosticV1::new(anchor, *carrier)); - } + let applications = carriers + .iter() + .filter_map(|reference| { + self.candidates + .get(reference) + .and_then(|candidate| candidate.header().application_header()) + }) + .filter(|header| self.included_applications.insert(header.reference())) + .cloned() + .collect(); self.pending_committed_frontiers .push(CommittedFrontierDeltaV1 { - output_sequence, anchor, frontier, carriers, applications, - application_diagnostics, }); - self.committed_output_count = next_count; - Ok(()) } fn ensure_live(&self) -> Result<(), ShadowErrorV1> { @@ -2072,10 +1483,7 @@ impl StarfishRbcDagShadowV1 { self.poisoned = true; return Err(error.into()); } - if let Err(error) = self.record_committed_input(&input, &effects) { - self.poisoned = true; - return Err(error); - } + self.record_committed_input(&input, &effects); Ok(effects) } @@ -2101,10 +1509,7 @@ impl StarfishRbcDagShadowV1 { self.poisoned = true; return Err(ShadowErrorV1::PostModelJournal(error)); } - if let Err(error) = self.record_committed_input(&input, &effects) { - self.poisoned = true; - return Err(error); - } + self.record_committed_input(&input, &effects); Ok(effects) } @@ -2178,17 +1583,11 @@ impl StarfishRbcDagShadowV1 { Ok(()) } - fn record_committed_input( - &mut self, - input: &ShadowInputV1, - effects: &[ModelEffect], - ) -> Result<(), ShadowErrorV1> { + fn record_committed_input(&mut self, input: &ShadowInputV1, effects: &[ModelEffect]) { if let Some(candidate) = input.candidate().cloned() { let reference = candidate.reference(); let slot = carrier_slot(reference); if let Some(vertex) = candidate.header().consensus_vertex() { - self.pending_promised_projection_candidates - .insert(reference); self.pending_projection_candidates.insert(reference); if input.is_local() { self.next_local_consensus_round = self @@ -2205,24 +1604,12 @@ impl StarfishRbcDagShadowV1 { self.projection .stage_carrier(candidate.clone()) .expect("durably validated carrier must match projection committee"); - self.promised_projection - .stage_carrier(candidate.clone()) - .expect("durably validated carrier must match promised projection committee"); - // Control carriers have no application materialization boundary, - // so the exact carrier bytes make them intrinsically available. - // An embedded application with an empty transaction commitment - // still needs its canonical header installed in Core before an - // authoritative frontier may reference it; the typed - // DataAvailable callback records that separate fact. - if candidate.header().application_header().is_none() - && candidate.header().transactions_commitment() == TransactionsCommitment::default() + if candidate.header().transactions_commitment() == TransactionsCommitment::default() + || input.is_local() { self.projection .mark_data_available(reference) .expect("staged control carrier is available"); - self.promised_projection - .mark_data_available(reference) - .expect("staged control carrier is available to promised projection"); } self.candidates.insert(reference, candidate); self.slot_candidates @@ -2247,9 +1634,6 @@ impl StarfishRbcDagShadowV1 { self.projection .mark_data_available(*reference) .expect("model accepted availability only for a staged carrier"); - self.promised_projection - .mark_data_available(*reference) - .expect("model accepted availability only for a promised staged carrier"); } for effect in effects { match effect { @@ -2257,22 +1641,16 @@ impl StarfishRbcDagShadowV1 { self.requested_recoveries.insert(*target, holders.clone()); } ModelEffect::Delivered(delivered) => { - self.certified_delivered.insert(*delivered); self.delivered.insert(*delivered); self.requested_recoveries.remove(delivered); self.projection .mark_delivered(*delivered) .expect("model delivery must name a staged carrier"); } - ModelEffect::DeliveryPromised(reference) => { - self.pending_promised_references.insert(*reference); - } ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} } } - self.activate_promised_references(); - self.drive_promised_projection(); - self.drive_certified_projection() + self.drive_certified_projection(); } fn decode_batch(&self, records: &[Vec]) -> Result { @@ -2533,14 +1911,6 @@ fn journal_transition_events( context, target: *target, }, - RbcPhaseStatementV1::Vote { target } => JournalEventV1::LockVote { - context, - target: *target, - }, - RbcPhaseStatementV1::Ack { target } => JournalEventV1::LockAck { - context, - target: *target, - }, }), ModelTraceEvent::PhaseBatchEntryApplied { outer, @@ -2590,17 +1960,6 @@ fn journal_transition_events( consensus_round: *consensus_round, choice: *choice, }), - ModelTraceEvent::DeliveryPromiseLocked { target, basis } => match basis { - DeliveryPromiseBasisV1::LocalFixed - | DeliveryPromiseBasisV1::HonestAuthor - | DeliveryPromiseBasisV1::OptimisticEcho => { - Some(JournalEventV1::LockOptimisticDelivery { - context, - target: *target, - }) - } - DeliveryPromiseBasisV1::Delivered => None, - }, ModelTraceEvent::DeliveryLocked(target) => Some(JournalEventV1::LockDelivery { context, target: *target, @@ -2800,7 +2159,6 @@ fn decode_raw_record( | RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY | RECORD_LOCAL_OUTBOUND_CONTENT - | RECORD_DATA_AVAILABLE | RECORD_MODEL_TRACE | RECORD_LOCAL_OUTBOUND_SIDECAR | RECORD_LOCAL_OUTBOUND_EXPOSE => {} @@ -2822,10 +2180,7 @@ fn decode_recorded_trace( let range = match decoded.first().map(|record| record.kind) { Some(RECORD_LOCAL_OUTBOUND_CONTENT) => 1..decoded.len().saturating_sub(2), Some( - RECORD_AUTHENTICATED_INGRESS - | RECORD_CANDIDATE_RETENTION - | RECORD_CANDIDATE_RECOVERY - | RECORD_DATA_AVAILABLE, + RECORD_AUTHENTICATED_INGRESS | RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY, ) => 1..decoded.len(), _ => return Err(ShadowErrorV1::InvalidBatch("missing model input")), }; @@ -2924,16 +2279,6 @@ fn encode_trace(trace: &ModelTraceEvent) -> Result, ShadowCodecErrorV1> bytes.extend_from_slice(&consensus_round.to_be_bytes()); push_leader_choice(&mut bytes, *choice); } - ModelTraceEvent::DeliveryPromiseLocked { target, basis } => { - bytes.push(TRACE_DELIVERY_PROMISE_LOCKED); - push_reference(&mut bytes, *target); - bytes.push(match basis { - DeliveryPromiseBasisV1::LocalFixed => 0, - DeliveryPromiseBasisV1::HonestAuthor => 1, - DeliveryPromiseBasisV1::OptimisticEcho => 2, - DeliveryPromiseBasisV1::Delivered => 3, - }); - } ModelTraceEvent::DeliveryLocked(reference) => { bytes.push(TRACE_DELIVERY_LOCKED); push_reference(&mut bytes, *reference); @@ -2974,16 +2319,6 @@ fn decode_trace( consensus_round: decoder.read_u32()?, choice: decoder.read_leader_choice()?, }, - TRACE_DELIVERY_PROMISE_LOCKED => ModelTraceEvent::DeliveryPromiseLocked { - target: decoder.read_reference()?, - basis: match decoder.read_u8()? { - 0 => DeliveryPromiseBasisV1::LocalFixed, - 1 => DeliveryPromiseBasisV1::HonestAuthor, - 2 => DeliveryPromiseBasisV1::OptimisticEcho, - 3 => DeliveryPromiseBasisV1::Delivered, - other => return Err(ShadowCodecErrorV1::InvalidDeliveryPromiseBasis(other)), - }, - }, TRACE_DELIVERY_LOCKED => ModelTraceEvent::DeliveryLocked(decoder.read_reference()?), TRACE_EFFECT => ModelTraceEvent::Effect(decoder.read_effect(committee_size)?), other => return Err(ShadowCodecErrorV1::InvalidTrace(other)), @@ -3008,10 +2343,6 @@ fn push_effect(bytes: &mut Vec, effect: &ModelEffect) -> Result<(), ShadowCo bytes.push(EFFECT_DELIVERED); push_reference(bytes, *reference); } - ModelEffect::DeliveryPromised(reference) => { - bytes.push(EFFECT_DELIVERY_PROMISED); - push_reference(bytes, *reference); - } ModelEffect::PrefixAdvanced { authority, tip } => { bytes.push(EFFECT_PREFIX_ADVANCED); bytes.extend_from_slice(&authority.to_be_bytes()); @@ -3035,14 +2366,6 @@ fn push_phase(bytes: &mut Vec, statement: RbcPhaseStatementV1) { bytes.push(PHASE_READY); push_reference(bytes, target); } - RbcPhaseStatementV1::Vote { target } => { - bytes.push(PHASE_VOTE); - push_reference(bytes, target); - } - RbcPhaseStatementV1::Ack { target } => { - bytes.push(PHASE_ACK); - push_reference(bytes, target); - } } } @@ -3172,108 +2495,6 @@ fn projection_decision_slot(decision: ProjectionDecisionV1) -> LeaderSlotV1 { } } -fn reconcile_authoritative_frontier_suffix( - replayed: Vec, - cursor: Option<&RbcDagFrontierRecoveryCursorV1>, - committee_size: usize, -) -> Result, ShadowErrorV1> { - if let Some(cursor) = cursor { - if cursor.receipt.committed_rounds.len() != committee_size { - return Err(ShadowErrorV1::FrontierRecoveryWatermarkLength { - expected: committee_size, - actual: cursor.receipt.committed_rounds.len(), - }); - } - } - - let mut committed_rounds = vec![0; committee_size]; - let mut last_sequence: RoundNumber = 0; - let mut cursor_found = cursor.is_none(); - let mut suffix = Vec::new(); - for delta in replayed { - let expected_sequence = - last_sequence - .checked_add(1) - .ok_or(ShadowErrorV1::FrontierOutputSequenceOverflow( - u64::from(last_sequence) + 1, - ))?; - if delta.output_sequence != expected_sequence { - return Err(ShadowErrorV1::FrontierRecoverySequence { - expected_sequence, - actual_sequence: delta.output_sequence, - }); - } - last_sequence = delta.output_sequence; - let consensus_round = delta.anchor.consensus_round(); - - let application_references = delta - .applications - .iter() - .map(RbcCanonicalHeader::reference) - .collect::>(); - for application in &application_references { - let Some(watermark) = committed_rounds.get_mut(application.authority as usize) else { - return Err(ShadowErrorV1::FrontierRecoveryApplicationAuthority { - application: *application, - committee_size, - }); - }; - *watermark = (*watermark).max(application.round); - } - - match cursor { - Some(cursor) if delta.output_sequence < cursor.receipt.output_sequence => {} - Some(cursor) if delta.output_sequence == cursor.receipt.output_sequence => { - if delta.anchor.carrier() != cursor.receipt.carrier_anchor { - return Err(ShadowErrorV1::FrontierRecoveryAnchorConflict { - consensus_round, - durable: cursor.receipt.carrier_anchor, - actor: delta.anchor.carrier(), - }); - } - if application_references != cursor.application_references { - return Err(ShadowErrorV1::FrontierRecoveryApplicationsConflict { - consensus_round, - durable: cursor.application_references.clone(), - actor: application_references, - }); - } - if committed_rounds != cursor.receipt.committed_rounds { - return Err(ShadowErrorV1::FrontierRecoveryWatermarksConflict { - consensus_round, - durable: cursor.receipt.committed_rounds.clone(), - actor: committed_rounds.clone(), - }); - } - cursor_found = true; - } - Some(_) => suffix.push(delta), - None => suffix.push(delta), - } - } - - if let Some(cursor) = cursor { - if !cursor_found { - if cursor.receipt.output_sequence > last_sequence { - return Err(ShadowErrorV1::FrontierRecoveryCursorAhead { - durable_sequence: cursor.receipt.output_sequence, - actor_sequence: last_sequence, - }); - } - return Err(ShadowErrorV1::FrontierRecoveryCursorMissing( - cursor.receipt.output_sequence, - )); - } - } - if suffix.len() > MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 { - return Err(ShadowErrorV1::FrontierRecoverySuffixLimit { - limit: MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1, - actual: suffix.len(), - }); - } - Ok(suffix) -} - struct RawDecoder<'a> { bytes: &'a [u8], position: usize, @@ -3335,8 +2556,6 @@ impl<'a> RawDecoder<'a> { match phase { PHASE_ECHO => Ok(RbcPhaseStatementV1::Echo { target }), PHASE_READY => Ok(RbcPhaseStatementV1::Ready { target }), - PHASE_VOTE => Ok(RbcPhaseStatementV1::Vote { target }), - PHASE_ACK => Ok(RbcPhaseStatementV1::Ack { target }), other => Err(ShadowCodecErrorV1::InvalidPhase(other)), } } @@ -3383,7 +2602,6 @@ impl<'a> RawDecoder<'a> { Ok(ModelEffect::NeedCarrier { target, holders }) } EFFECT_DELIVERED => Ok(ModelEffect::Delivered(self.read_reference()?)), - EFFECT_DELIVERY_PROMISED => Ok(ModelEffect::DeliveryPromised(self.read_reference()?)), EFFECT_PREFIX_ADVANCED => Ok(ModelEffect::PrefixAdvanced { authority: self.read_u16()?, tip: self.read_reference()?, @@ -3426,9 +2644,7 @@ mod tests { MAC_TAG_SIZE, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }, - starfish_rbc_dag::{ - RbcDagProtocolInstanceId, carrier_genesis_reference, journal::RbcSlotKeyV1, - }, + starfish_rbc_dag::{RbcDagProtocolInstanceId, carrier_genesis_reference}, }; const N: usize = 4; @@ -3480,13 +2696,8 @@ mod tests { self.directories[authority].path().join("shadow.wal") } - fn run_four_phase_rounds_with_one_poisoned_recipient(&mut self) { - // INIT/ECHO is exposed in round one, the accumulated ECHOs drive - // VOTE+ACK in round two, ACK convergence drives READY in round - // three, and the Q-READY certificate is observed in round four. - // Keep these as distinct physical carrier rounds: collapsing the - // final transition would fail to exercise the runtime backlog. - for round in 1..=4 { + fn run_three_rounds_with_one_poisoned_recipient(&mut self) { + for round in 1..=3 { let envelopes = self .nodes .iter_mut() @@ -3540,403 +2751,10 @@ mod tests { } } - fn ordered_committer_vertex( - author: AuthorityIndex, - consensus_round: RoundNumber, - variant: u8, - ) -> ConsensusVertexReference { - let marker = (consensus_round as u8) - .wrapping_mul(17) - .wrapping_add(author as u8) - .wrapping_add(variant.wrapping_mul(71)); - ConsensusVertexReference::new( - BlockReference { - authority: author, - round: 100 + consensus_round * 2 + RoundNumber::from(variant), - digest: BlockDigest::from([marker; 32]), - }, - consensus_round, - ) - } - - fn quorum_authors_including( - committee: &RbcDagCommitteeContextV1, - required: AuthorityIndex, - ) -> Vec { - std::iter::once(required) - .chain( - committee - .committee() - .authorities() - .filter(|author| *author != required), - ) - .take(3) - .collect() - } - - fn inject_ordered_committer_fixture( - node: &mut StarfishRbcDagShadowV1, - include_early_direct_anchor: bool, - ) -> [ConsensusVertexReference; 3] { - let committee = node.committee.clone(); - let leader_author = |round: RoundNumber| committee.committee().elect_leader(round); - let no_vote = |round: RoundNumber| LeaderChoiceV1::NoVote { - leader_author: leader_author(round - 1), - leader_round: round - 1, - }; - let older = ordered_committer_vertex(leader_author(1), 1, 0); - node.projection - .inject_projected_for_test(older, Vec::new(), no_vote(1)); - - // Q voters make the round-one leader certifiable, but only one - // round-three vertex initially carries that certificate. Direct - // commit is therefore unavailable while indirect commit is possible. - let round_two = (0..N as AuthorityIndex) - .map(|author| ordered_committer_vertex(author, 2, 0)) - .collect::>(); - for (author, reference) in round_two.iter().copied().enumerate() { - let choice = if author < 3 { - LeaderChoiceV1::Vote { leader: older } - } else { - no_vote(2) - }; - node.projection - .inject_projected_for_test(reference, vec![older], choice); - } - - let round_three = (0..3 as AuthorityIndex) - .map(|author| ordered_committer_vertex(author, 3, 0)) - .collect::>(); - let round_three_parents = [ - round_two[..3].to_vec(), - vec![round_two[0], round_two[1], round_two[3]], - vec![round_two[1], round_two[2], round_two[3]], - ]; - for (reference, parents) in round_three.iter().copied().zip(round_three_parents) { - node.projection - .inject_projected_for_test(reference, parents, no_vote(3)); - } - - // The first later leader is reachable from the single certificate. - // It is initially only indirectly committed by the still-later - // leader, which is the case the old direct-only anchor scan skipped. - let first_anchor = ordered_committer_vertex(leader_author(4), 4, 0); - let round_four_authors = quorum_authors_including(&committee, first_anchor.author()); - let mut round_four = Vec::new(); - for author in round_four_authors { - let reference = if author == first_anchor.author() { - first_anchor - } else { - ordered_committer_vertex(author, 4, 0) - }; - node.projection - .inject_projected_for_test(reference, round_three.clone(), no_vote(4)); - round_four.push(reference); - } - - let round_five = (0..N as AuthorityIndex) - .map(|author| ordered_committer_vertex(author, 5, 0)) - .collect::>(); - for (author, reference) in round_five.iter().copied().enumerate() { - let choice = if author < 3 { - LeaderChoiceV1::Vote { - leader: first_anchor, - } - } else { - no_vote(5) - }; - node.projection - .inject_projected_for_test(reference, round_four.clone(), choice); - } - - let round_six = (0..3 as AuthorityIndex) - .map(|author| ordered_committer_vertex(author, 6, 0)) - .collect::>(); - let round_six_parents = [ - round_five[..3].to_vec(), - vec![round_five[0], round_five[1], round_five[3]], - vec![round_five[1], round_five[2], round_five[3]], - ]; - for (reference, parents) in round_six.iter().copied().zip(round_six_parents) { - node.projection - .inject_projected_for_test(reference, parents, no_vote(6)); - } - if include_early_direct_anchor { - // One Byzantine author supplies a conflicting certifier while the - // unused fourth author supplies another. Together with author 0, - // direct evidence for the round-four leader reaches Q. - for (author, variant) in [(1, 1), (3, 0)] { - node.projection.inject_projected_for_test( - ordered_committer_vertex(author, 6, variant), - round_five[..3].to_vec(), - no_vote(6), - ); - } - } - - let later_anchor = ordered_committer_vertex(leader_author(7), 7, 0); - let round_seven_authors = quorum_authors_including(&committee, later_anchor.author()); - let mut round_seven = Vec::new(); - for author in round_seven_authors { - let reference = if author == later_anchor.author() { - later_anchor - } else { - ordered_committer_vertex(author, 7, 0) - }; - node.projection - .inject_projected_for_test(reference, round_six.clone(), no_vote(7)); - round_seven.push(reference); - } - let round_eight = (0..3 as AuthorityIndex) - .map(|author| ordered_committer_vertex(author, 8, 0)) - .collect::>(); - for reference in &round_eight { - node.projection.inject_projected_for_test( - *reference, - round_seven.clone(), - LeaderChoiceV1::Vote { - leader: later_anchor, - }, - ); - } - for author in 0..3 as AuthorityIndex { - node.projection.inject_projected_for_test( - ordered_committer_vertex(author, 9, 0), - round_eight.clone(), - no_vote(9), - ); - } - [older, first_anchor, later_anchor] - } - - #[test] - fn ordered_committer_is_deterministic_across_direct_anchor_arrival_orders() { - let mut network = TestNetwork::new(); - let expected = inject_ordered_committer_fixture(&mut network.nodes[0], false); - assert_eq!( - inject_ordered_committer_fixture(&mut network.nodes[1], true), - expected - ); - - network.nodes[0].drive_ordered_committer(9).unwrap(); - network.nodes[1].drive_ordered_committer(9).unwrap(); - - let delayed_decisions = network.nodes[0].drain_projection_decisions(); - let eager_decisions = network.nodes[1].drain_projection_decisions(); - assert!( - delayed_decisions.contains(&ProjectionDecisionV1::IndirectCommit { - leader: expected[1], - anchor: expected[2], - }) - ); - assert!( - eager_decisions.contains(&ProjectionDecisionV1::DirectCommit { - leader: expected[1], - }) - ); - - let delayed_output = network.nodes[0].drain_committed_frontiers(); - let eager_output = network.nodes[1].drain_committed_frontiers(); - assert_eq!(delayed_output, eager_output); - assert_eq!( - delayed_output - .iter() - .map(|delta| delta.anchor) - .collect::>(), - expected - ); - assert_eq!( - delayed_output - .iter() - .map(|delta| delta.output_sequence) - .collect::>(), - vec![1, 2, 3] - ); - } - - #[test] - fn vote_and_ack_trace_codec_preserves_append_only_golden_tags() { - let target = BlockReference { - authority: 0x0123, - round: 0x0405_0607, - digest: BlockDigest::from([0xA5; 32]), - }; - let mut vote_golden = vec![ - TRACE_LOCAL_PHASE_LOCKED, - PHASE_VOTE, - 0x01, - 0x23, - 0x04, - 0x05, - 0x06, - 0x07, - ]; - vote_golden.extend_from_slice(&[0xA5; 32]); - let mut ack_golden = vote_golden.clone(); - ack_golden[1] = PHASE_ACK; - - let vote = ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Vote { target }); - let ack = ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Ack { target }); - assert_eq!(encode_trace(&vote).unwrap(), vote_golden); - assert_eq!(encode_trace(&ack).unwrap(), ack_golden); - assert_eq!(decode_trace(&vote_golden, N).unwrap(), vote); - assert_eq!(decode_trace(&ack_golden, N).unwrap(), ack); - - let batch_payload = encode_trace_batch(&[vote.clone(), ack.clone()]).unwrap(); - let committee = Committee::new_test(vec![1; N]); - let committee = RbcDagCommitteeContextV1::new(committee).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xD4; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ); - let raw = encode_raw_record(context, 0, RECORD_MODEL_TRACE, &batch_payload).unwrap(); - assert_eq!(&raw[..4], b"SRD5"); - let decoded = decode_raw_record(&raw, context, 0).unwrap(); - assert_eq!(decoded.kind, RECORD_MODEL_TRACE); - assert_eq!( - decode_trace_batch(&decoded.payload, N).unwrap(), - vec![vote, ack] - ); - } - - #[test] - fn vote_and_ack_local_locks_and_pending_phases_survive_shadow_reopen() { - let mut network = TestNetwork::new(); - let target = round_one_candidate(0, &network.committee, 0xD5); - let target_reference = target.reference(); - let authentication = network - .context - .authenticate_with_committee( - &target, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 0, - keys: &network.keyrings[0], - }, - ) - .unwrap(); - network.nodes[3] - .receive_authenticated_from_peer( - &target.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - 0, - ) - .unwrap(); - - let outer = round_two_phase_carrier( - 1, - RbcPhaseStatementV1::Echo { - target: target_reference, - }, - &network.committee, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - network.nodes[3] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - 1, - ) - .unwrap(); - - // Phase statements for a round-one target become eligible only once - // the local physical clock opens round two. Fix the local round-one - // carrier and admit one more round-one peer to expose the exact - // pending VOTE/ACK batch before restart. - network.nodes[3] - .create_local_control_heartbeat(10, true) - .unwrap(); - let peer = round_one_candidate(2, &network.committee, 0xD6); - let authentication = network - .context - .authenticate_with_committee( - &peer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 2, - keys: &network.keyrings[2], - }, - ) - .unwrap(); - network.nodes[3] - .receive_authenticated_from_peer( - &peer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - 2, - ) - .unwrap(); - assert_eq!(network.nodes[3].local_carrier_round(), 2); - - let expected = [ - RbcPhaseStatementV1::Vote { - target: target_reference, - }, - RbcPhaseStatementV1::Ack { - target: target_reference, - }, - ]; - let slot = RbcSlotKeyV1::of(target_reference); - for phase in expected { - let pending = network.nodes[3].model.pending_phase_batch(); - assert!( - pending.contains(&phase), - "missing {phase:?} from pending phase batch {pending:?}" - ); - } - assert_eq!( - network.nodes[3].journal.snapshot().vote_lock(slot), - Some(target_reference) - ); - assert_eq!( - network.nodes[3].journal.snapshot().ack_lock(slot), - Some(target_reference) - ); - - let node = network.nodes.swap_remove(3); - let path = network.path(3); - node.shutdown().unwrap(); - let (reopened, report) = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 3, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[3].clone()), - ) - .unwrap(); - assert!(report.replayed_batches() >= 2); - for phase in expected { - let pending = reopened.model.pending_phase_batch(); - assert!( - pending.contains(&phase), - "reopen lost {phase:?} from pending phase batch {pending:?}" - ); - } - assert_eq!( - reopened.journal.snapshot().vote_lock(slot), - Some(target_reference) - ); - assert_eq!( - reopened.journal.snapshot().ack_lock(slot), - Some(target_reference) - ); - reopened.shutdown().unwrap(); - } - #[test] - fn four_phase_mac_shadow_delivers_round_one_after_poisoned_tag_is_only_staged() { + fn three_round_mac_shadow_delivers_round_one_after_poisoned_tag_is_only_staged() { let mut network = TestNetwork::new(); - network.run_four_phase_rounds_with_one_poisoned_recipient(); + network.run_three_rounds_with_one_poisoned_recipient(); for node in &network.nodes { for author in 0..N { @@ -3967,20 +2785,7 @@ mod tests { let before = node.wal_counts(); let (heartbeat, effects) = node.create_local_control_heartbeat(123, true).unwrap(); - assert_eq!( - effects, - vec![ - ModelEffect::DeliveryPromised(heartbeat.reference()), - ModelEffect::PrefixAdvanced { - authority: 0, - tip: heartbeat.reference(), - }, - ] - ); - assert_eq!( - node.model.delivery_promise_basis(&heartbeat.reference()), - Some(DeliveryPromiseBasisV1::LocalFixed) - ); + assert!(effects.is_empty()); assert_eq!(node.wal_counts().0, before.0 + 1); let candidate = decode_candidate( heartbeat.canonical_carrier_wire(), @@ -4020,11 +2825,7 @@ mod tests { assert_eq!(node.admitted_reference(0, 1), Some(heartbeat.reference())); assert_eq!(node.current_round_admitted_author_count(), 1); assert_eq!(node.current_round_admitted_stake(), 1); - assert_eq!( - node.pending_phase_backlog_len(), - 0, - "the target author is excluded from ECHO/VOTE/ACK" - ); + assert_eq!(node.pending_phase_backlog_len(), 1); assert_eq!(node.buffered_authenticated_carrier_count(), 0); let durable_counts = node.wal_counts(); @@ -4042,506 +2843,30 @@ mod tests { } #[test] - fn optimistic_delivery_projects_before_q_ready_for_the_same_locked_vertex() { + fn autonomous_control_heartbeat_advances_sequentially_and_reopens_exact_bytes() { let mut network = TestNetwork::new(); - let target = round_one_consensus_candidate( - 1, - &network.committee, - 0x91, - TransactionsCommitment::from_bytes([0x91; 32]), - ); - let target_reference = target.reference(); - let vertex_reference = ConsensusVertexReference::new(target_reference, 1); - let authentication = network - .context - .authenticate_with_committee( - &target, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - assert!( + let first = network.nodes[0] + .create_local_control_heartbeat(1_000, true) + .unwrap() + .0; + for author in [1, 2] { + let candidate = round_one_candidate(author, &network.committee, 0x70 + author as u8); + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: author, + keys: &network.keyrings[author as usize], + }, + ) + .unwrap(); network.nodes[0] .receive_authenticated_from_peer( - &target.canonical_wire_bytes().unwrap(), + &candidate.canonical_wire_bytes().unwrap(), &authentication.canonical_wire_bytes(), - 1, - ) - .unwrap() - .is_empty() - ); - - // The target author's ECHO is excluded from the optimistic - // certificate. Receiving only that statement cannot promise. - let sender = 1; - { - let outer = phase_carrier( - sender, - 2, - RbcPhaseStatementV1::Echo { - target: target_reference, - }, - &network.committee, - 0xA0 + sender as u8, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: sender, - keys: &network.keyrings[sender as usize], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - sender, - ) - .unwrap(); - } - assert_eq!( - network.nodes[0] - .model - .delivery_promise_basis(&target_reference), - None - ); - assert!( - !network.nodes[0] - .promised_projection - .is_projected(vertex_reference) - ); - assert!(!network.nodes[0].projection.is_projected(vertex_reference)); - - // The receiver's local ECHO plus one other non-author ECHO reaches - // the N=4 optimistic threshold O=2. This is deliberately earlier - // than the later Q-READY delivery certificate. - let sender = 2; - let outer = phase_carrier( - sender, - 2, - RbcPhaseStatementV1::Echo { - target: target_reference, - }, - &network.committee, - 0xA2, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: sender, - keys: &network.keyrings[sender as usize], - }, - ) - .unwrap(); - let effects = network.nodes[0] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - sender, - ) - .unwrap(); - assert!(effects.contains(&ModelEffect::DeliveryPromised(target_reference))); - assert_eq!( - network.nodes[0] - .model - .delivery_promise_basis(&target_reference), - Some(DeliveryPromiseBasisV1::OptimisticEcho) - ); - assert_eq!(network.nodes[0].promised_projection.promised_tip(1), None); - assert!( - !network.nodes[0] - .promised_projection - .is_projected(vertex_reference) - ); - assert!(!network.nodes[0].carrier_data_available(target_reference)); - - network.nodes[0] - .mark_carrier_data_available(target_reference) - .unwrap(); - assert!( - network.nodes[0] - .promised_projection - .is_projected(vertex_reference) - ); - assert_eq!( - network.nodes[0].promised_projection.promised_tip(1), - Some(target_reference) - ); - assert!(network.nodes[0].projection.is_projected(vertex_reference)); - assert_eq!( - network.nodes[0].projection.closed_tip(1), - Some(target_reference) - ); - assert_eq!( - network.nodes[0].drain_projected_vertices(), - vec![vertex_reference] - ); - assert!(network.nodes[0].drain_projection_decisions().is_empty()); - assert!(network.nodes[0].drain_committed_frontiers().is_empty()); - - // Reopening from typed inputs and trace effects reconstructs the - // authoritative optimistic delivery and the identical planning view. - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let mut node = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap() - .0; - assert!(node.promised_projection.is_projected(vertex_reference)); - assert!(node.projection.is_projected(vertex_reference)); - assert!( - !node - .model - .lifecycle(&target_reference) - .unwrap() - .certified_delivered - ); - assert!(node.drain_committed_frontiers().is_empty()); - - // Two external READYs first trigger the local READY and then produce - // the final Q-READY delivery. Certified projection catches up to the - // exact immutable vertex and effective frontier planned earlier. - for sender in [1, 2] { - let outer = phase_carrier( - sender, - 3, - RbcPhaseStatementV1::Ready { - target: target_reference, - }, - &network.committee, - 0xB0 + sender as u8, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: sender, - keys: &network.keyrings[sender as usize], - }, - ) - .unwrap(); - node.receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - sender, - ) - .unwrap(); - } - assert!(node.model.lifecycle(&target_reference).unwrap().delivered); - assert!( - node.model - .lifecycle(&target_reference) - .unwrap() - .certified_delivered - ); - assert!(node.projection.is_projected(vertex_reference)); - assert_eq!( - node.promised_projection.projected_vertex(vertex_reference), - node.projection.projected_vertex(vertex_reference) - ); - assert_eq!( - node.promised_projection - .effective_frontier(vertex_reference), - node.projection.effective_frontier(vertex_reference) - ); - for projected in node.drain_projected_vertices() { - assert!(node.projection.is_projected(projected)); - } - for delta in node.drain_committed_frontiers() { - assert!(node.projection.is_projected(delta.anchor)); - assert!(delta.carriers.iter().all(|reference| { - node.model.lifecycle(reference).is_some_and(|lifecycle| { - lifecycle.delivered && lifecycle.data_available && lifecycle.prefix_closed - }) - })); - } - } - - #[test] - fn promise_for_missing_content_waits_for_exact_recovery_without_panicking() { - let mut network = TestNetwork::new(); - let target = round_one_consensus_candidate( - 1, - &network.committee, - 0x92, - TransactionsCommitment::default(), - ); - let target_reference = target.reference(); - let vertex_reference = ConsensusVertexReference::new(target_reference, 1); - let unrelated = round_one_candidate(2, &network.committee, 0x93); - - // This directly exercises the durable-adapter boundary represented by - // an all-author-ECHO promise whose exact carrier bytes have not yet - // arrived. The effect is retained, not applied to a placeholder. - network.nodes[0] - .record_committed_input( - &ShadowInputV1::CandidateRetention(unrelated), - &[ModelEffect::DeliveryPromised(target_reference)], - ) - .unwrap(); - assert!( - network.nodes[0] - .pending_promised_references - .contains(&target_reference) - ); - assert!( - !network.nodes[0] - .promised_projection - .is_projected(vertex_reference) - ); - - network.nodes[0] - .record_committed_input(&ShadowInputV1::CandidateRecovery(target), &[]) - .unwrap(); - assert!(network.nodes[0].pending_promised_references.is_empty()); - assert!( - network.nodes[0] - .promised_projection - .is_projected(vertex_reference) - ); - assert!(network.nodes[0].projection.is_projected(vertex_reference)); - } - - #[test] - fn missing_content_echo_evidence_reopens_then_promises_on_exact_recovery() { - let mut network = TestNetwork::new(); - let target = round_one_consensus_candidate( - 3, - &network.committee, - 0x94, - TransactionsCommitment::default(), - ); - let target_reference = target.reference(); - let vertex_reference = ConsensusVertexReference::new(target_reference, 1); - - // Three remote ECHOs reach Q while exact content is missing. The - // receiver cannot count its own ECHO without first authenticating the - // carrier, so the reducer requests recovery but emits no promise. - for sender in [1, 2, 3] { - let outer = phase_carrier( - sender, - 2, - RbcPhaseStatementV1::Echo { - target: target_reference, - }, - &network.committee, - 0xC0 + sender as u8, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: sender, - keys: &network.keyrings[sender as usize], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - sender, - ) - .unwrap(); - } - assert!( - network.nodes[0] - .retained_candidate_wire(target_reference) - .is_none() - ); - assert_eq!( - network.nodes[0] - .model - .delivery_promise_basis(&target_reference), - None - ); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let (mut node, report) = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap(); - assert!(report.recovery_effects().iter().any(|effect| { - matches!(effect, ModelEffect::NeedCarrier { target, .. } if *target == target_reference) - })); - assert_eq!(node.model.delivery_promise_basis(&target_reference), None); - - let recovery_effects = node - .recover_candidate_for(target_reference, &target.canonical_wire_bytes().unwrap()) - .unwrap(); - assert!(node.retained_candidate_wire(target_reference).is_some()); - assert!( - recovery_effects.contains(&ModelEffect::DeliveryPromised(target_reference)), - "the persisted ECHO certificate must activate once exact content arrives" - ); - assert_eq!( - node.model.delivery_promise_basis(&target_reference), - Some(DeliveryPromiseBasisV1::OptimisticEcho) - ); - assert!(node.promised_projection.is_projected(vertex_reference)); - - // Later receiver-authenticated ingress may authorize the local ECHO, - // but it must not duplicate the already durable promise. - let authentication = network - .context - .authenticate_with_committee( - &target, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 3, - keys: &network.keyrings[3], - }, - ) - .unwrap(); - let effects = node - .receive_authenticated_from_peer( - &target.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - 3, - ) - .unwrap(); - assert!(!effects.contains(&ModelEffect::DeliveryPromised(target_reference))); - assert!(node.promised_projection.is_projected(vertex_reference)); - assert!(node.projection.is_projected(vertex_reference)); - assert!( - !node - .model - .lifecycle(&target_reference) - .unwrap() - .certified_delivered - ); - } - - #[test] - fn c1_builder_waits_for_the_witness_and_avoids_the_mixed_slow_tail() { - let committee = Committee::new_test(vec![1; 7]); - let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xC1; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ); - let keyrings = mac_keyrings_for_test(7); - let directory = tempfile::tempdir().unwrap(); - let mut node = StarfishRbcDagShadowV1::open( - directory.path().join("c1-builder.wal"), - committee.clone(), - 0, - context, - ShadowAuthorizerV1::MacVector(keyrings[0].clone()), - ) - .unwrap() - .0; - let reference = |authority, consensus_round, marker| { - ConsensusVertexReference::new( - BlockReference { - authority, - round: consensus_round + 100, - digest: BlockDigest::from([marker; 32]), - }, - consensus_round, - ) - }; - - let target = reference(committee.committee().elect_leader(2), 2, 0x20); - node.promised_projection.inject_projected_for_test( - target, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: committee.committee().elect_leader(1), - leader_round: 1, - }, - ); - for author in [0, 1, 2, 3, 6] { - let projected = reference(author, 3, 0x30 + author as u8); - let choice = if author == 6 { - LeaderChoiceV1::NoVote { - leader_author: target.author(), - leader_round: target.consensus_round(), - } - } else { - LeaderChoiceV1::Vote { leader: target } - }; - node.promised_projection - .inject_projected_for_test(projected, vec![target], choice); - } - node.next_local_consensus_round = 4; - assert!( - node.build_local_consensus_vertex(carrier_genesis_reference(0), false) - .is_none(), - "a mixed first quorum must not fix a non-certifying C1 vertex" - ); - - let final_voter = reference(4, 3, 0x34); - node.promised_projection.inject_projected_for_test( - final_voter, - vec![target], - LeaderChoiceV1::Vote { leader: target }, - ); - let vertex = node - .build_local_consensus_vertex(carrier_genesis_reference(0), false) - .expect("the exact fifth vote completes C1"); - let authors = vertex - .strong_parents() - .iter() - .map(|parent| parent.author()) - .collect::>(); - assert_eq!(authors, vec![0, 1, 2, 3, 4]); - assert_eq!(vertex.strong_parents().len(), 5); - assert!(!authors.contains(&6)); - } - - #[test] - fn autonomous_control_heartbeat_advances_sequentially_and_reopens_exact_bytes() { - let mut network = TestNetwork::new(); - let first = network.nodes[0] - .create_local_control_heartbeat(1_000, true) - .unwrap() - .0; - for author in [1, 2] { - let candidate = round_one_candidate(author, &network.committee, 0x70 + author as u8); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: author, - keys: &network.keyrings[author as usize], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &candidate.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - author, + author, ) .unwrap(); } @@ -4601,152 +2926,6 @@ mod tests { assert!(restarted.journal.snapshot().leader_choice(1).is_some()); } - #[test] - fn empty_application_waits_for_explicit_core_materialization() { - let mut network = TestNetwork::new(); - let application_header = RbcCanonicalHeader::try_new( - 0, - 1, - network - .committee - .committee() - .authorities() - .map(carrier_genesis_reference) - .collect(), - Vec::new(), - 899, - TransactionsCommitment::default(), - ) - .unwrap(); - let reference = network.nodes[0] - .create_local_application_carrier(application_header, 999, true) - .unwrap() - .0 - .reference(); - - assert!( - network.nodes[0] - .model - .lifecycle(&reference) - .unwrap() - .data_available, - "the empty payload is intrinsically available to the RBC reducer" - ); - assert!( - !network.nodes[0].projection.is_data_available(reference), - "authoritative projection must still wait for the Core header" - ); - - network.nodes[0] - .mark_carrier_data_available(reference) - .unwrap(); - assert!(network.nodes[0].projection.is_data_available(reference)); - assert!( - network.nodes[0] - .promised_projection - .is_data_available(reference) - ); - } - - #[test] - fn application_data_availability_record_reopens_from_the_wal() { - let mut network = TestNetwork::new(); - let commitment = TransactionsCommitment::from_bytes([0xDA; 32]); - let application_header = RbcCanonicalHeader::try_new( - 0, - 1, - network - .committee - .committee() - .authorities() - .map(carrier_genesis_reference) - .collect(), - Vec::new(), - 900, - commitment, - ) - .unwrap(); - let reference = network.nodes[0] - .create_local_application_carrier(application_header, 1_000, true) - .unwrap() - .0 - .reference(); - assert!(!network.nodes[0].carrier_data_available(reference)); - assert_eq!(network.nodes[0].projection.closed_tip(0), None); - - for sender in [1, 2] { - let outer = round_two_phase_carrier( - sender, - RbcPhaseStatementV1::Ready { target: reference }, - &network.committee, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: sender, - keys: &network.keyrings[sender as usize], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - sender, - ) - .unwrap(); - } - - let vertex = ConsensusVertexReference::new(reference, 1); - let lifecycle = network.nodes[0].model.lifecycle(&reference).unwrap(); - assert!(lifecycle.delivered); - assert!(!lifecycle.prefix_closed); - assert_eq!(network.nodes[0].projection.closed_tip(0), None); - assert!(matches!( - network.nodes[0].projection.try_project(reference), - Err(CertifiedProjectionError::CarrierDataUnavailable(actual)) if actual == reference - )); - - let effects = network.nodes[0] - .mark_carrier_data_available(reference) - .unwrap(); - assert!(effects.iter().any( - |effect| matches!(effect, ModelEffect::PrefixAdvanced { tip, .. } if *tip == reference) - )); - assert!(network.nodes[0].carrier_data_available(reference)); - assert!( - network.nodes[0] - .model - .lifecycle(&reference) - .unwrap() - .prefix_closed - ); - assert_eq!(network.nodes[0].projection.closed_tip(0), Some(reference)); - assert!(network.nodes[0].projection.is_projected(vertex)); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let (restarted, report) = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap(); - - assert_eq!(report.replayed_batches(), 4); - assert!(restarted.carrier_data_available(reference)); - assert!(restarted.model.lifecycle(&reference).unwrap().prefix_closed); - assert_eq!(restarted.projection.closed_tip(0), Some(reference)); - assert!(restarted.projection.is_projected(vertex)); - assert!(restarted.retained_candidate_wire(reference).is_some()); - } - #[test] fn authenticated_replays_and_slot_conflicts_do_not_grow_durable_state() { let mut network = TestNetwork::new(); @@ -4853,18 +3032,18 @@ mod tests { } #[test] - fn ignored_far_future_ingress_skips_authentication_and_durable_state() { + fn rejected_far_future_ingress_does_not_poison_the_durable_actor() { let mut network = TestNetwork::new(); let author = 1; let previous = |authority: AuthorityIndex| BlockReference { authority, - round: 65, + round: 5, digest: BlockDigest::from([0x90 + authority as u8; 32]), }; let candidate = CandidateCarrierV1::try_new_with_committee( CarrierHeaderV1Args { author, - carrier_round: 66, + carrier_round: 6, own_prev: previous(author), weak_parents: [0, 2].into_iter().map(previous).collect(), transactions_commitment: TransactionsCommitment::default(), @@ -4877,20 +3056,32 @@ mod tests { &network.committee, ) .unwrap(); - let outcome = network.nodes[0] - .receive_or_retain_from_peer( - &candidate.canonical_wire_bytes().unwrap(), - // The far-future prefilter must run before parsing or - // verifying the authentication sidecar. - b"not-an-authentication-sidecar", - author, + let authentication = network + .context + .authenticate_with_committee( + &candidate, + &network.committee, + CarrierAuthorizerV1::MacVector { + authority: author, + keys: &network.keyrings[author as usize], + }, ) .unwrap(); - assert_eq!( - outcome.disposition(), - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer - ); - assert!(outcome.effects().is_empty()); + + assert!(matches!( + network.nodes[0].receive_or_retain_from_peer( + &candidate.canonical_wire_bytes().unwrap(), + &authentication.canonical_wire_bytes(), + author, + ), + Err(ShadowErrorV1::Model( + ModelError::FutureCarrierOutsideBuffer { + current: 1, + maximum: 5, + actual: 6, + } + )) + )); assert_eq!(network.nodes[0].wal_counts(), (0, 0)); network.nodes[0] .create_local_control_heartbeat(2, true) @@ -5044,12 +3235,12 @@ mod tests { #[test] fn wal_restart_past_round_one_discards_torn_tail_and_retransmits_exact_bytes() { let mut network = TestNetwork::new(); - network.run_four_phase_rounds_with_one_poisoned_recipient(); + network.run_three_rounds_with_one_poisoned_recipient(); let node = network.nodes.swap_remove(0); let path = network.path(0); let before = node.retransmissions(); - assert_eq!(before.len(), 4); + assert_eq!(before.len(), 3); let retained = node.retained_candidate_wire(before[0].reference()).unwrap(); assert_eq!(retained, before[0].canonical_carrier_wire()); node.shutdown().unwrap(); @@ -5087,7 +3278,7 @@ mod tests { .unwrap(); assert_eq!(report.discarded_tail_bytes(), torn.len() as u64); assert!(report.replayed_batches() > 3); - assert_eq!(restarted.local_carrier_round(), 5); + assert_eq!(restarted.local_carrier_round(), 4); assert_eq!(restarted.retransmissions(), before); assert!(report.recovery_effects().is_empty()); for author in 0..N { @@ -5240,7 +3431,7 @@ mod tests { #[test] fn direct_shadow_comparison_reports_match_mismatch_and_ambiguity_without_references() { let mut network = TestNetwork::new(); - network.run_four_phase_rounds_with_one_poisoned_recipient(); + network.run_three_rounds_with_one_poisoned_recipient(); let node = &network.nodes[0]; let direct = node.delivered_identities().unwrap(); assert_eq!( @@ -5362,91 +3553,6 @@ mod tests { .unwrap() } - fn round_one_consensus_candidate( - author: AuthorityIndex, - committee: &RbcDagCommitteeContextV1, - marker: u8, - transactions_commitment: TransactionsCommitment, - ) -> CandidateCarrierV1 { - let weak_parents = committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(carrier_genesis_reference) - .collect(); - let strong_parents = committee - .committee() - .authorities() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - let leader_author = committee.committee().elect_leader(0); - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 1, - own_prev: carrier_genesis_reference(author), - weak_parents, - transactions_commitment, - application_header: None, - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: Some(ConsensusVertexV1::new( - 1, - strong_parents, - vec![None; committee.committee().len()], - LeaderChoiceV1::Vote { - leader: ConsensusVertexReference::new( - carrier_genesis_reference(leader_author), - 0, - ), - }, - )), - creation_time_ns: u64::from(marker), - }, - committee, - ) - .unwrap() - } - - fn phase_carrier( - author: AuthorityIndex, - round: RoundNumber, - statement: RbcPhaseStatementV1, - committee: &RbcDagCommitteeContextV1, - marker: u8, - ) -> CandidateCarrierV1 { - assert!(round > 1); - let previous = |authority: AuthorityIndex| BlockReference { - authority, - round: round - 1, - digest: BlockDigest::from([marker.wrapping_add(authority as u8); 32]), - }; - let weak_parents = committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(previous) - .collect(); - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: round, - own_prev: previous(author), - weak_parents, - transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - application_header: None, - data_acknowledgments: Vec::new(), - phase_batch: vec![statement], - consensus_vertex: None, - creation_time_ns: u64::from(round), - }, - committee, - ) - .unwrap() - } - fn round_two_phase_carrier( author: AuthorityIndex, statement: RbcPhaseStatementV1, @@ -5483,173 +3589,4 @@ mod tests { ) .unwrap() } - - fn recovery_application( - authority: AuthorityIndex, - round: RoundNumber, - marker: u8, - ) -> RbcCanonicalHeader { - RbcCanonicalHeader::try_new( - authority, - round, - (0..N as AuthorityIndex) - .map(carrier_genesis_reference) - .collect(), - Vec::new(), - u64::from(marker), - TransactionsCommitment::from_bytes([marker; 32]), - ) - .unwrap() - } - - fn recovery_delta( - output_sequence: RoundNumber, - consensus_round: RoundNumber, - applications: Vec, - ) -> CommittedFrontierDeltaV1 { - let carrier = BlockReference::new_test( - (consensus_round as usize % N) as AuthorityIndex, - consensus_round.saturating_add(100), - ); - let application_diagnostics = applications - .iter() - .map(|_| CommittedApplicationDiagnosticV1 { - physical_carrier_round_delta: 0, - }) - .collect(); - CommittedFrontierDeltaV1 { - output_sequence, - anchor: ConsensusVertexReference::new(carrier, consensus_round), - frontier: vec![None; N], - carriers: Vec::new(), - applications, - application_diagnostics, - } - } - - fn recovery_cursor( - delta: &CommittedFrontierDeltaV1, - committed_rounds: Vec, - ) -> RbcDagFrontierRecoveryCursorV1 { - RbcDagFrontierRecoveryCursorV1 { - receipt: RbcDagFrontierReceipt { - carrier_anchor: delta.anchor.carrier(), - output_sequence: delta.output_sequence, - committed_rounds, - }, - application_references: delta - .applications - .iter() - .map(RbcCanonicalHeader::reference) - .collect(), - } - } - - #[test] - fn authoritative_frontier_recovery_replays_only_the_exact_newer_suffix() { - let first_application = recovery_application(1, 5, 0xA1); - let last_application = recovery_application(2, 7, 0xA2); - // Logical anchor rounds may regress while the output sequence remains - // contiguous and monotone. - let first = recovery_delta(1, 8, vec![first_application]); - let control_only = recovery_delta(2, 3, Vec::new()); - let last = recovery_delta(3, 7, vec![last_application]); - let history = vec![first.clone(), control_only.clone(), last.clone()]; - - assert_eq!( - reconcile_authoritative_frontier_suffix(history.clone(), None, N).unwrap(), - history - ); - - let after_first = reconcile_authoritative_frontier_suffix( - history.clone(), - Some(&recovery_cursor(&first, vec![0, 5, 0, 0])), - N, - ) - .unwrap(); - assert_eq!(after_first, vec![control_only.clone(), last.clone()]); - - let after_control = reconcile_authoritative_frontier_suffix( - history, - Some(&recovery_cursor(&control_only, vec![0, 5, 0, 0])), - N, - ) - .unwrap(); - assert_eq!(after_control, vec![last]); - } - - #[test] - fn authoritative_frontier_recovery_rejects_unreconciled_cursors() { - let application = recovery_application(1, 5, 0xB1); - let first = recovery_delta(1, 8, vec![application]); - let last = recovery_delta(2, 3, Vec::new()); - let history = vec![first.clone(), last.clone()]; - - let mut conflict = recovery_cursor(&first, vec![0, 5, 0, 0]); - conflict.receipt.carrier_anchor = BlockReference::new_test(0, 999); - assert!(matches!( - reconcile_authoritative_frontier_suffix(history.clone(), Some(&conflict), N), - Err(ShadowErrorV1::FrontierRecoveryAnchorConflict { .. }) - )); - - let mut conflict = recovery_cursor(&first, vec![0, 5, 0, 0]); - conflict.application_references.clear(); - assert!(matches!( - reconcile_authoritative_frontier_suffix(history.clone(), Some(&conflict), N), - Err(ShadowErrorV1::FrontierRecoveryApplicationsConflict { .. }) - )); - - let conflict = recovery_cursor(&first, vec![0, 4, 0, 0]); - assert!(matches!( - reconcile_authoritative_frontier_suffix(history.clone(), Some(&conflict), N), - Err(ShadowErrorV1::FrontierRecoveryWatermarksConflict { .. }) - )); - - let missing = RbcDagFrontierRecoveryCursorV1 { - receipt: RbcDagFrontierReceipt { - carrier_anchor: BlockReference::new_test(2, 103), - output_sequence: 3, - committed_rounds: vec![0, 5, 0, 0], - }, - application_references: Vec::new(), - }; - assert!(matches!( - reconcile_authoritative_frontier_suffix(history.clone(), Some(&missing), N), - Err(ShadowErrorV1::FrontierRecoveryCursorAhead { - durable_sequence: 3, - actor_sequence: 2 - }) - )); - - let ahead = RbcDagFrontierRecoveryCursorV1 { - receipt: RbcDagFrontierReceipt { - carrier_anchor: BlockReference::new_test(0, 104), - output_sequence: 4, - committed_rounds: vec![0, 5, 0, 0], - }, - application_references: Vec::new(), - }; - assert!(matches!( - reconcile_authoritative_frontier_suffix(history, Some(&ahead), N), - Err(ShadowErrorV1::FrontierRecoveryCursorAhead { - durable_sequence: 4, - actor_sequence: 2 - }) - )); - } - - #[test] - fn authoritative_frontier_recovery_suffix_is_bounded() { - let history = (1..=MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 + 1) - .map(|sequence| { - recovery_delta(sequence as RoundNumber, sequence as RoundNumber, Vec::new()) - }) - .collect(); - assert!(matches!( - reconcile_authoritative_frontier_suffix(history, None, N), - Err(ShadowErrorV1::FrontierRecoverySuffixLimit { limit, actual }) - if limit == MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 - && actual == MAX_AUTHORITATIVE_FRONTIER_RECOVERY_SUFFIX_V1 + 1 - )); - } } diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index d4504394..7aff59da 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -20,46 +20,31 @@ use parking_lot::Mutex; use tokio::{ sync::{ mpsc::{self, error::TrySendError}, - oneshot, watch, + oneshot, }, task::JoinHandle, }; use crate::{ - crypto::{ - MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE, - TransactionsCommitment, - }, - metrics::{ - Metrics, RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, - }, + crypto::{MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE}, network::{ - NetworkMessage, RbcDagApplicationPayloadResponse, RbcDagShadowCarrier, - RbcDagShadowCarrierResponse, RbcDagShadowCarrierSyncRequest, - RbcDagShadowCarrierSyncResponse, + NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse, + RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ - CandidateCarrierV1, ConsensusVertexReference, MAX_CARRIER_CONTENT_SIZE_V1, - RbcDagCommitteeContextV1, RbcDagContextV1, - model::{ - EXECUTABLE_MODEL_ADMISSION_WINDOW_V1, EXECUTABLE_MODEL_BUFFER_WINDOW_V1, ModelEffect, - ModelError, - }, + ConsensusVertexReference, MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, + RbcDagContextV1, + model::{ModelEffect, ModelError}, projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ - CommittedFrontierDeltaV1, RbcDagFrontierRecoveryCursorV1, ShadowAuthorizerV1, - ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, ShadowDeliverySlotV1, ShadowErrorV1, - ShadowIngressDispositionV1, ShadowOpenReportV1, ShadowOutboundEnvelopeV1, - StarfishRbcDagShadowV1, - }, - types::{ - AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs, - TransactionData, + CommittedFrontierDeltaV1, ShadowAuthorizerV1, ShadowDeliveryComparisonV1, + ShadowDeliveryIdentityV1, ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, + ShadowOpenReportV1, ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, }, + types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, }; // A mirror run must absorb one complete committee fan-in plus a small reserve; @@ -80,58 +65,7 @@ const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); const SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1: Duration = Duration::from_millis(100); -// At most sixteen distinct exact slots may bypass the duplicate retry interval -// per requester. Even at the four-MiB carrier ceiling this bounds one -// interval's replay exposure to 64 MiB; ordinary network backpressure remains -// the second bound. Production should replace this prototype credit with a -// configured byte-rate budget. -const SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1: usize = 16; -/// Exact repair is an independently bounded priority lane. The same cap is -/// used for outstanding request slots and for responses coalesced outside the -/// ordinary actor FIFO. -const SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1: usize = 64; const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_millis(500); -/// This prototype applies the existing four-MiB canonical-content/default -/// block ceiling as a conservative serialized-payload preflight. The network -/// frame ceiling is larger; a dedicated configurable payload limit remains a -/// deployment-hardening boundary. Keeping only a bounded recent window stops -/// unsolicited sidecars from turning the actor into an unbounded cache. -const SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1: usize = SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1; -const SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1: usize = MAX_CARRIER_CONTENT_SIZE_V1; -const SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); -/// Bound healthy physical-carrier production independently of actor/network -/// scheduling. The 600-ms Starfish pacemaker therefore permits at most about -/// 33 normal carriers/s, while validity-backed repair retains its separately -/// bounded burst lane. -const SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1: u32 = 20; -const SHADOW_NORMAL_CARRIER_MIN_SPACING_V1: Duration = Duration::from_millis(1); - -type CarrierSyncSlotV1 = (RoundNumber, AuthorityIndex); -type DesiredCarrierSyncResponseV1 = (AuthorityIndex, RbcDagShadowCarrierSyncResponse); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct NormalCarrierDeadlineV1 { - generation: u64, - deadline: Instant, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct ConsensusTimeoutDeadlineV1 { - generation: u64, - slot: RoundNumber, - deadline: Instant, -} - -#[cfg(test)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct CarrierSyncInspectionV1 { - open_round: RoundNumber, - target: Option, - outstanding: usize, - desired_responses: usize, - max_outstanding: usize, - max_desired_responses: usize, -} /// Runtime role of the persisted carrier actor. /// @@ -166,138 +100,29 @@ impl ShadowServiceModeV1 { } } -/// Per-logical-slot Starfish creation pacemaker. -/// -/// The physical heartbeat runs on a fixed grid, so its tick time cannot also -/// be the C2 origin: a logical slot may have opened only an instant before the -/// tick. C2 is armed once A1 is locally true for this exact slot. C3 remains -/// an immediate, independently sufficient catch-up condition. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct ConsensusPacemakerV1 { - slot: RoundNumber, - c2_armed_at: Option, - c2_timed_out: bool, -} - -impl ConsensusPacemakerV1 { - fn new(slot: RoundNumber) -> Self { - Self { - slot, - c2_armed_at: None, - c2_timed_out: false, - } - } - - fn fallback_allowed( - &mut self, - slot: RoundNumber, - a1_ready: bool, - c3_ready: bool, - leader_timeout: Duration, - now: Instant, - ) -> bool { - if self.slot != slot { - self.slot = slot; - self.c2_armed_at = None; - self.c2_timed_out = false; - } - if a1_ready { - self.c2_armed_at.get_or_insert(now); - } else { - // Eligible projection is monotonic, so this is principally a - // fail-closed guard against arming from the wrong logical slot. - self.c2_armed_at = None; - self.c2_timed_out = false; - } - c3_ready - || self.c2_timed_out - || self - .c2_armed_at - .is_some_and(|armed| now.saturating_duration_since(armed) >= leader_timeout) - } - - fn observe_timeout(&mut self, slot: RoundNumber) -> bool { - if self.slot != slot || self.c2_armed_at.is_none() { - return false; - } - self.c2_timed_out = true; - true - } -} - -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] struct ShadowLocalCarrierV1 { author: AuthorityIndex, round: RoundNumber, - transactions_commitment: TransactionsCommitment, + transactions_commitment: crate::crypto::TransactionsCommitment, creation_time_ns: TimestampNs, application_header: RbcCanonicalHeader, - application_payload: Option>, - /// True only when the live core submitted this header and is waiting for - /// an `ApplicationAssigned` flow-control acknowledgment. Recovered direct - /// history may be assigned during WAL reconciliation without a live - /// producer gate to release. - acknowledge_assignment: bool, } impl ShadowLocalCarrierV1 { - #[cfg(test)] fn from_direct_header(header: &RbcCanonicalHeader) -> Self { - Self::from_direct_header_with_payload(header, None) - } - - fn from_direct_header_with_payload( - header: &RbcCanonicalHeader, - application_payload: Option>, - ) -> Self { - Self::from_direct_header_with_ack(header, application_payload, true) - } - - fn from_recovered_direct_header(header: &RbcCanonicalHeader) -> Self { - Self::from_direct_header_with_ack(header, None, false) - } - - fn from_direct_header_with_ack( - header: &RbcCanonicalHeader, - application_payload: Option>, - acknowledge_assignment: bool, - ) -> Self { Self { author: header.reference().authority, round: header.reference().round, transactions_commitment: header.transactions_commitment(), creation_time_ns: header.meta_creation_time_ns(), application_header: header.clone(), - application_payload, - acknowledge_assignment, } } - - fn same_application(&self, other: &Self) -> bool { - self.author == other.author - && self.round == other.round - && self.transactions_commitment == other.transactions_commitment - && self.creation_time_ns == other.creation_time_ns - && self.application_header == other.application_header - } -} - -impl PartialEq for ShadowLocalCarrierV1 { - fn eq(&self, other: &Self) -> bool { - self.same_application(other) - && application_payloads_equal( - self.application_payload.as_deref(), - other.application_payload.as_deref(), - ) - && self.acknowledge_assignment == other.acknowledge_assignment - } } -impl Eq for ShadowLocalCarrierV1 {} - enum ShadowServiceMessageV1 { - ActivateClock(oneshot::Sender<()>), - LocalApplicationsChanged, + LocalCarrier(ShadowLocalCarrierV1), Carrier { peer: AuthorityIndex, envelope: RbcDagShadowCarrier, @@ -314,32 +139,15 @@ enum ShadowServiceMessageV1 { peer: AuthorityIndex, request: RbcDagShadowCarrierSyncRequest, }, - CarrierSyncResponsesChanged, - ApplicationPayloadRequest { - peer: AuthorityIndex, - application: BlockReference, - }, - ApplicationPayloadResponse { + CarrierSyncResponse { peer: AuthorityIndex, - response: RbcDagApplicationPayloadResponse, + response: RbcDagShadowCarrierSyncResponse, }, - VerifiedApplicationPayloadsChanged, DirectDeliveriesChanged, TopologyChanged, RetryRecovery, HeartbeatTick, - NormalCarrierDeadline { - generation: u64, - }, - ConsensusTimeoutDeadline { - generation: u64, - slot: RoundNumber, - }, DataAvailabilityChanged, - #[cfg(test)] - InspectRbcProgress(oneshot::Sender<(usize, usize)>), - #[cfg(test)] - InspectCarrierSync(oneshot::Sender), Shutdown(oneshot::Sender>), } @@ -349,25 +157,15 @@ pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { max_sidecar_size: usize, own_authority: AuthorityIndex, committee_size: usize, - #[cfg(test)] input_capacity: usize, mode: ShadowServiceModeV1, desired_topology: Arc>>, - desired_local_applications: Arc>>, - /// Exact `(round, author)` responses outside the ordinary actor FIFO. - /// Catch-up must not wait behind the proactive future carriers it is - /// intended to overtake. - desired_carrier_sync_responses: - Arc>>, - desired_verified_application_payloads: - Arc>>>, desired_direct_deliveries: Arc>>, desired_available_applications: Arc>>, invalidated_by_overload: Arc>>, } impl StarfishRbcDagShadowServiceHandleV1 { - #[cfg(test)] fn send(&self, message: ShadowServiceMessageV1) -> Result<(), ShadowServiceErrorV1> { let kind = message.kind(); if let Some(reason) = *self.invalidated_by_overload.lock() { @@ -385,72 +183,15 @@ impl StarfishRbcDagShadowServiceHandleV1 { }) } - async fn send_reliably( - &self, - message: ShadowServiceMessageV1, - ) -> Result<(), ShadowServiceErrorV1> { - if let Some(reason) = *self.invalidated_by_overload.lock() { - return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); - } - self.sender - .send(message) - .await - .map_err(|_| ShadowServiceErrorV1::Stopped) - } - pub(crate) fn local_header( &self, header: &RbcCanonicalHeader, ) -> Result<(), ShadowServiceErrorV1> { - self.local_application(header, None) - } - - /// Coalescing producer boundary for a standalone embedded application. - /// The optional payload is availability data only; its header remains the - /// sole identity and is commitment-checked inside the actor before use. - pub(crate) fn local_application( - &self, - header: &RbcCanonicalHeader, - application_payload: Option>, - ) -> Result<(), ShadowServiceErrorV1> { - if let Some(reason) = *self.invalidated_by_overload.lock() { - return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); - } - if let Some(payload) = &application_payload { - validate_application_payload_size(payload)?; - } - let local = - ShadowLocalCarrierV1::from_direct_header_with_payload(header, application_payload); - let mut desired = self.desired_local_applications.lock(); - if let Some(existing) = desired.get_mut(&local.round) { - if !existing.same_application(&local) { - return Err(ShadowServiceErrorV1::ConflictingLocalHeader(local.round)); - } - merge_application_payload( - &mut existing.application_payload, - local.application_payload, - existing.application_header.reference(), - )?; - existing.acknowledge_assignment |= local.acknowledge_assignment; - return Ok(()); - } - if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - return Err(ShadowServiceErrorV1::ApplicationStateCapacity { - capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, - }); - } - desired.insert(local.round, local); - drop(desired); - match self - .sender - .try_send(ShadowServiceMessageV1::LocalApplicationsChanged) - { - Ok(()) | Err(TrySendError::Full(_)) => Ok(()), - Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), - } + self.send(ShadowServiceMessageV1::LocalCarrier( + ShadowLocalCarrierV1::from_direct_header(header), + )) } - #[cfg(test)] pub(crate) fn carrier( &self, peer: AuthorityIndex, @@ -466,35 +207,9 @@ impl StarfishRbcDagShadowServiceHandleV1 { envelope.authentication_sidecar.len(), self.max_sidecar_size, )?; - if let Some(payload) = &envelope.application_payload { - validate_application_payload_size(payload)?; - } self.send(ShadowServiceMessageV1::Carrier { peer, envelope }) } - pub(crate) async fn carrier_reliably( - &self, - peer: AuthorityIndex, - envelope: RbcDagShadowCarrier, - ) -> Result<(), ShadowServiceErrorV1> { - validate_wire_size( - "carrier", - envelope.canonical_carrier.len(), - MAX_CARRIER_CONTENT_SIZE_V1, - )?; - validate_wire_size( - "authentication sidecar", - envelope.authentication_sidecar.len(), - self.max_sidecar_size, - )?; - if let Some(payload) = &envelope.application_payload { - validate_application_payload_size(payload)?; - } - self.send_reliably(ShadowServiceMessageV1::Carrier { peer, envelope }) - .await - } - - #[cfg(test)] pub(crate) fn carrier_request( &self, peer: AuthorityIndex, @@ -503,16 +218,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierRequest { peer, reference }) } - pub(crate) async fn carrier_request_reliably( - &self, - peer: AuthorityIndex, - reference: BlockReference, - ) -> Result<(), ShadowServiceErrorV1> { - self.send_reliably(ShadowServiceMessageV1::CarrierRequest { peer, reference }) - .await - } - - #[cfg(test)] pub(crate) fn carrier_response( &self, peer: AuthorityIndex, @@ -526,21 +231,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierResponse { peer, response }) } - pub(crate) async fn carrier_response_reliably( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierResponse, - ) -> Result<(), ShadowServiceErrorV1> { - validate_wire_size( - "carrier response", - response.canonical_carrier.len(), - MAX_CARRIER_CONTENT_SIZE_V1, - )?; - self.send_reliably(ShadowServiceMessageV1::CarrierResponse { peer, response }) - .await - } - - #[cfg(test)] pub(crate) fn carrier_sync_request( &self, peer: AuthorityIndex, @@ -552,46 +242,14 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierSyncRequest { peer, request }) } - pub(crate) async fn carrier_sync_request_reliably( - &self, - peer: AuthorityIndex, - request: RbcDagShadowCarrierSyncRequest, - ) -> Result<(), ShadowServiceErrorV1> { - if !self.mode.is_autonomous() { - return Ok(()); - } - self.send_reliably(ShadowServiceMessageV1::CarrierSyncRequest { peer, request }) - .await - } - - #[cfg(test)] pub(crate) fn carrier_sync_response( &self, peer: AuthorityIndex, response: RbcDagShadowCarrierSyncResponse, - ) -> Result<(), ShadowServiceErrorV1> { - self.enqueue_carrier_sync_response(peer, response) - } - - pub(crate) async fn carrier_sync_response_reliably( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierSyncResponse, - ) -> Result<(), ShadowServiceErrorV1> { - self.enqueue_carrier_sync_response(peer, response) - } - - fn enqueue_carrier_sync_response( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierSyncResponse, ) -> Result<(), ShadowServiceErrorV1> { if !self.mode.is_autonomous() { return Ok(()); } - if peer as usize >= self.committee_size { - return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); - } validate_wire_size( "carrier sync response", response.canonical_carrier.len(), @@ -602,116 +260,7 @@ impl StarfishRbcDagShadowServiceHandleV1 { response.authentication_sidecar.len(), self.max_sidecar_size, )?; - if response.author != peer { - return Err(ShadowServiceErrorV1::UnexpectedSyncResponse { - author: response.author, - round: response.round, - }); - } - - let slot = (response.round, response.author); - let mut desired = self.desired_carrier_sync_responses.lock(); - match desired.get(&slot) { - Some((existing_peer, existing)) if *existing_peer == peer && *existing == response => { - return Ok(()); - } - Some(_) => { - return Err(ShadowServiceErrorV1::UnexpectedSyncResponse { - author: response.author, - round: response.round, - }); - } - None if desired.len() >= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 => { - return Err(ShadowServiceErrorV1::CarrierSyncResponseCapacity { - capacity: SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, - }); - } - None => {} - } - desired.insert(slot, (peer, response)); - drop(desired); - match self - .sender - .try_send(ShadowServiceMessageV1::CarrierSyncResponsesChanged) - { - Ok(()) | Err(TrySendError::Full(_)) => Ok(()), - Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), - } - } - - #[cfg(test)] - pub(crate) fn application_payload_request( - &self, - peer: AuthorityIndex, - application: BlockReference, - ) -> Result<(), ShadowServiceErrorV1> { - self.send(ShadowServiceMessageV1::ApplicationPayloadRequest { peer, application }) - } - - pub(crate) async fn application_payload_request_reliably( - &self, - peer: AuthorityIndex, - application: BlockReference, - ) -> Result<(), ShadowServiceErrorV1> { - self.send_reliably(ShadowServiceMessageV1::ApplicationPayloadRequest { peer, application }) - .await - } - - #[cfg(test)] - pub(crate) fn application_payload_response( - &self, - peer: AuthorityIndex, - response: RbcDagApplicationPayloadResponse, - ) -> Result<(), ShadowServiceErrorV1> { - validate_application_payload_size(&response.transaction_data)?; - self.send(ShadowServiceMessageV1::ApplicationPayloadResponse { peer, response }) - } - - pub(crate) async fn application_payload_response_reliably( - &self, - peer: AuthorityIndex, - response: RbcDagApplicationPayloadResponse, - ) -> Result<(), ShadowServiceErrorV1> { - validate_application_payload_size(&response.transaction_data)?; - self.send_reliably(ShadowServiceMessageV1::ApplicationPayloadResponse { peer, response }) - .await - } - - /// Trusted callback from the sole transaction-commitment verifier. The - /// actor accepts it only for an already-authorized application reference; - /// network payload bytes can never call this path directly. - pub(crate) fn verified_application_payload( - &self, - application: BlockReference, - payload: Arc, - ) -> Result<(), ShadowServiceErrorV1> { - if let Some(reason) = *self.invalidated_by_overload.lock() { - return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); - } - validate_application_payload_size(&payload)?; - let mut desired = self.desired_verified_application_payloads.lock(); - if let Some(existing) = desired.get(&application) { - if !application_payloads_equal(Some(existing.as_ref()), Some(payload.as_ref())) { - return Err(ShadowServiceErrorV1::ConflictingApplicationPayload( - application, - )); - } - return Ok(()); - } - if desired.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - return Err(ShadowServiceErrorV1::ApplicationStateCapacity { - capacity: SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1, - }); - } - desired.insert(application, payload); - drop(desired); - match self - .sender - .try_send(ShadowServiceMessageV1::VerifiedApplicationPayloadsChanged) - { - Ok(()) | Err(TrySendError::Full(_)) => Ok(()), - Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), - } + self.send(ShadowServiceMessageV1::CarrierSyncResponse { peer, response }) } pub(crate) fn direct_delivered( @@ -784,39 +333,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped)? } - /// Release a coordinated autonomous-clock start barrier. This operation - /// is idempotent: after the first successful activation, later callers - /// receive an acknowledgment without resetting the heartbeat epoch. - pub(crate) async fn activate_clock(&self) -> Result<(), ShadowServiceErrorV1> { - if !self.mode.is_autonomous() { - return Ok(()); - } - let (reply, receiver) = oneshot::channel(); - self.send_reliably(ShadowServiceMessageV1::ActivateClock(reply)) - .await?; - receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped) - } - - #[cfg(test)] - async fn inspect_rbc_progress(&self) -> Result<(usize, usize), ShadowServiceErrorV1> { - let (reply, receiver) = oneshot::channel(); - self.sender - .send(ShadowServiceMessageV1::InspectRbcProgress(reply)) - .await - .map_err(|_| ShadowServiceErrorV1::Stopped)?; - receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped) - } - - #[cfg(test)] - async fn inspect_carrier_sync(&self) -> Result { - let (reply, receiver) = oneshot::channel(); - self.sender - .send(ShadowServiceMessageV1::InspectCarrierSync(reply)) - .await - .map_err(|_| ShadowServiceErrorV1::Stopped)?; - receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped) - } - fn update_peer( &self, peer: AuthorityIndex, @@ -846,52 +362,29 @@ impl StarfishRbcDagShadowServiceHandleV1 { } impl ShadowServiceMessageV1 { - #[cfg(test)] fn kind(&self) -> &'static str { match self { - Self::ActivateClock(_) => "activate_clock", - Self::LocalApplicationsChanged => "local_applications_changed", + Self::LocalCarrier(_) => "local", Self::Carrier { .. } => "carrier", Self::CarrierRequest { .. } => "carrier_request", Self::CarrierResponse { .. } => "carrier_response", Self::CarrierSyncRequest { .. } => "carrier_sync_request", - Self::CarrierSyncResponsesChanged => "carrier_sync_responses_changed", - Self::ApplicationPayloadRequest { .. } => "application_payload_request", - Self::ApplicationPayloadResponse { .. } => "application_payload_response", - Self::VerifiedApplicationPayloadsChanged => "verified_application_payloads_changed", + Self::CarrierSyncResponse { .. } => "carrier_sync_response", Self::DirectDeliveriesChanged => "direct_deliveries_changed", Self::TopologyChanged => "topology_changed", Self::RetryRecovery => "recovery_retry", Self::HeartbeatTick => "heartbeat_tick", - Self::NormalCarrierDeadline { .. } => "normal_carrier_deadline", - Self::ConsensusTimeoutDeadline { .. } => "consensus_timeout_deadline", Self::DataAvailabilityChanged => "data_availability_changed", - Self::InspectRbcProgress(_) => "inspect_rbc_progress", - Self::InspectCarrierSync(_) => "inspect_carrier_sync", Self::Shutdown(_) => "shutdown", } } } -/// Exact protocol fact permitting an embedded application header to leave the -/// shadow actor. Payload availability is deliberately absent from this enum: -/// bytes can accompany authority, but can never create it. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ShadowApplicationAuthorizationBasisV1 { - LocallyFixed, - ReceiverAuthenticated, - Delivered, -} - #[derive(Debug)] pub(crate) enum ShadowServiceEventV1 { Ready { autonomous_clock: bool, }, - /// The autonomous actor has crossed its one-way coordinated-start - /// barrier. Local carrier creation and exact repair are enabled, and the - /// first physical heartbeat is one full interval after this event. - ClockActivated, ClockState { open_round: RoundNumber, phase_backlog: usize, @@ -913,21 +406,6 @@ pub(crate) enum ShadowServiceEventV1 { carrier: BlockReference, header: RbcCanonicalHeader, }, - /// An application header whose enclosing carrier has an exact local, - /// receiver-authenticated, or delivered authorization basis. A later - /// repeat may enrich an earlier header-only event with its verified - /// transaction payload. - AuthorizedApplicationObserved { - carrier: BlockReference, - header: RbcCanonicalHeader, - payload: Option>, - authorization_basis: ShadowApplicationAuthorizationBasisV1, - }, - /// The exact direct application header has been durably assigned to one - /// local carrier. In embedded-authority mode this is the flow-control - /// acknowledgment that permits the core to produce the next application - /// header; it is not a delivery or commit certificate. - ApplicationAssigned(BlockReference), VertexProjected(ConsensusVertexReference), LeaderDecided(ProjectionDecisionV1), FrontierCommitted(CommittedFrontierDeltaV1), @@ -957,7 +435,6 @@ pub(crate) enum ShadowServiceErrorV1 { Shadow(ShadowErrorV1), StartTask(tokio::task::JoinError), Stopped, - #[cfg(test)] Overloaded { kind: &'static str, capacity: usize, @@ -978,18 +455,6 @@ pub(crate) enum ShadowServiceErrorV1 { UnknownAuthority(AuthorityIndex), Loopback(AuthorityIndex), ConflictingLocalHeader(RoundNumber), - ConflictingApplicationPayload(BlockReference), - ApplicationStateCapacity { - capacity: usize, - }, - CarrierSyncResponseCapacity { - capacity: usize, - }, - ApplicationPayloadSerialization(String), - ApplicationPayloadResponseFromUnexpectedPeer { - peer: AuthorityIndex, - application: BlockReference, - }, MissingRecoveredLocalHeader(RoundNumber), RecoveredLocalHeaderMismatch(RoundNumber), AutonomousWalContainsInvalidCarrier(RoundNumber), @@ -1029,7 +494,6 @@ impl fmt::Display for ShadowServiceErrorV1 { "Starfish-RBC-DAG shadow startup task failed: {error}" ), Self::Stopped => formatter.write_str("Starfish-RBC-DAG shadow service stopped"), - #[cfg(test)] Self::Overloaded { kind, capacity } => write!( formatter, "Starfish-RBC-DAG shadow {kind} input was dropped because the queue is full \ @@ -1066,26 +530,6 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "conflicting direct headers supplied for queued shadow round {round}" ), - Self::ConflictingApplicationPayload(application) => write!( - formatter, - "conflicting transaction payloads supplied for embedded application {application}" - ), - Self::ApplicationStateCapacity { capacity } => write!( - formatter, - "embedded application state reached its bounded capacity of {capacity} entries" - ), - Self::CarrierSyncResponseCapacity { capacity } => write!( - formatter, - "exact carrier-sync response state reached its bounded capacity of {capacity} slots" - ), - Self::ApplicationPayloadSerialization(error) => write!( - formatter, - "embedded application payload could not be size-checked: {error}" - ), - Self::ApplicationPayloadResponseFromUnexpectedPeer { peer, application } => write!( - formatter, - "embedded application payload response for {application} came from unrequested authority {peer}" - ), Self::MissingRecoveredLocalHeader(round) => write!( formatter, "persisted shadow carrier at round {round} has no matching recovered direct header" @@ -1152,7 +596,6 @@ impl From for ShadowServiceErrorV1 { } } -#[cfg(test)] pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, @@ -1178,22 +621,18 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( recovered_local_headers, ShadowServiceModeV1::DirectMirror, wal_sync_policy, - None, - None, - true, ) } -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( +pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, own_authority: AuthorityIndex, context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, + heartbeat_interval: Duration, wal_sync_policy: ShadowWalSyncPolicyV1, - metrics: Arc, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -1202,6 +641,9 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( ), ShadowServiceErrorV1, > { + if heartbeat_interval.is_zero() { + return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); + } start_starfish_rbc_dag_shadow_service_with_mode_v1( path, committee, @@ -1209,237 +651,11 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_with_metrics_v1( context, authorizer, recovered_local_headers, - ShadowServiceModeV1::DirectMirror, + ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, wal_sync_policy, - Some(metrics), - None, - true, ) } -#[cfg(test)] -pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, - heartbeat_interval: Duration, - wal_sync_policy: ShadowWalSyncPolicyV1, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, -> { - if heartbeat_interval.is_zero() { - return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); - } - start_starfish_rbc_dag_shadow_service_with_mode_v1( - path, - committee, - own_authority, - context, - authorizer, - recovered_local_headers, - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - wal_sync_policy, - None, - None, - true, - ) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, - heartbeat_interval: Duration, - wal_sync_policy: ShadowWalSyncPolicyV1, - metrics: Arc, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, -> { - if heartbeat_interval.is_zero() { - return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); - } - start_starfish_rbc_dag_shadow_service_with_mode_v1( - path, - committee, - own_authority, - context, - authorizer, - recovered_local_headers, - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - wal_sync_policy, - Some(metrics), - None, - true, - ) -} - -/// Start an autonomous runtime whose protocol clock remains paused after WAL -/// open/replay and `Ready`. The caller must invoke -/// [`StarfishRbcDagShadowServiceHandleV1::activate_clock`] after its external -/// startup barrier is satisfied. Ordinary start APIs remain active-by-default. -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_paused_with_metrics_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, - heartbeat_interval: Duration, - wal_sync_policy: ShadowWalSyncPolicyV1, - metrics: Arc, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, -> { - if heartbeat_interval.is_zero() { - return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); - } - start_starfish_rbc_dag_shadow_service_with_mode_v1( - path, - committee, - own_authority, - context, - authorizer, - recovered_local_headers, - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - wal_sync_policy, - Some(metrics), - None, - false, - ) -} - -/// Start the embedded-authority runtime with Core's exact durable recovery -/// cursor. Unlike the compatibility autonomous wrappers, this is the -/// production restart API: it fails closed unless the actor WAL reconciles -/// the cursor and its unapplied suffix is bounded. -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_starfish_rbc_dag_authoritative_clock_service_with_metrics_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, - heartbeat_interval: Duration, - wal_sync_policy: ShadowWalSyncPolicyV1, - metrics: Arc, - recovery_cursor: Option, - clock_starts_active: bool, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, -> { - if heartbeat_interval.is_zero() { - return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); - } - start_starfish_rbc_dag_shadow_service_with_mode_v1( - path, - committee, - own_authority, - context, - authorizer, - recovered_local_headers, - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - wal_sync_policy, - Some(metrics), - recovery_cursor, - clock_starts_active, - ) -} - -fn spawn_consensus_timeout_deadline_task( - mut deadline_rx: watch::Receiver>, - timeout_tx: mpsc::WeakSender, -) { - tokio::spawn(async move { - loop { - if deadline_rx.changed().await.is_err() { - return; - } - let Some(mut target) = *deadline_rx.borrow_and_update() else { - continue; - }; - 'scheduled: loop { - tokio::select! { - _ = tokio::time::sleep_until(tokio::time::Instant::from_std(target.deadline)) => { - if *deadline_rx.borrow() != Some(target) { - break 'scheduled; - } - let Some(sender) = timeout_tx.upgrade() else { - return; - }; - let message = ShadowServiceMessageV1::ConsensusTimeoutDeadline { - generation: target.generation, - slot: target.slot, - }; - // A full actor FIFO must not pin a strong sender after - // the actor cancels/replaces this deadline or exits. - // The watch branch aborts the pending send; any wake - // already queued is rejected by its generation. - tokio::select! { - biased; - changed = deadline_rx.changed() => { - if changed.is_err() { - return; - } - match *deadline_rx.borrow_and_update() { - Some(replacement) => { - target = replacement; - continue 'scheduled; - } - None => break 'scheduled, - } - } - sent = sender.send(message) => { - if sent.is_err() { - return; - } - break 'scheduled; - } - } - } - changed = deadline_rx.changed() => { - if changed.is_err() { - return; - } - match *deadline_rx.borrow_and_update() { - Some(replacement) => target = replacement, - None => break 'scheduled, - } - } - } - } - } - }); -} - fn start_starfish_rbc_dag_shadow_service_with_mode_v1( path: impl AsRef, committee: RbcDagCommitteeContextV1, @@ -1449,9 +665,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( recovered_local_headers: Vec, mode: ShadowServiceModeV1, wal_sync_policy: ShadowWalSyncPolicyV1, - metrics: Option>, - recovery_cursor: Option, - clock_starts_active: bool, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -1467,7 +680,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let path = path.as_ref().to_path_buf(); let mut pending_local = BTreeMap::new(); for header in recovered_local_headers { - let local = ShadowLocalCarrierV1::from_recovered_direct_header(&header); + let local = ShadowLocalCarrierV1::from_direct_header(&header); if local.author != own_authority { return Err(ShadowServiceErrorV1::LocalHeaderAuthority { expected: own_authority, @@ -1484,24 +697,11 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let (message_tx, message_rx) = mpsc::channel(input_capacity); let (event_tx, event_rx) = mpsc::channel(SHADOW_SERVICE_EVENT_CAPACITY_V1); let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); - let desired_local_applications = Arc::new(Mutex::new(BTreeMap::new())); - let desired_carrier_sync_responses = Arc::new(Mutex::new(BTreeMap::new())); - let desired_verified_application_payloads = Arc::new(Mutex::new(BTreeMap::new())); let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); let desired_available_applications = Arc::new(Mutex::new(BTreeSet::new())); let invalidated_by_overload = Arc::new(Mutex::new(None)); let retry_notification_pending = Arc::new(AtomicBool::new(false)); let heartbeat_notification_pending = Arc::new(AtomicBool::new(false)); - // The physical heartbeat is deliberately created only after the actor - // crosses its one-way activation barrier. In paused benchmark startup, - // validators can therefore spend arbitrarily different amounts of time - // opening WALs and establishing topology without inheriting staggered - // timer phases or accumulating missed ticks. - let (clock_activation_tx, mut clock_activation_rx) = watch::channel(false); - let (normal_carrier_deadline_tx, mut normal_carrier_deadline_rx) = - watch::channel(None::); - let (consensus_timeout_deadline_tx, consensus_timeout_deadline_rx) = - watch::channel(None::); let retry_tx = message_tx.downgrade(); let retry_pending = Arc::clone(&retry_notification_pending); tokio::spawn(async move { @@ -1530,16 +730,12 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let heartbeat_tx = message_tx.downgrade(); let heartbeat_pending = Arc::clone(&heartbeat_notification_pending); tokio::spawn(async move { - if clock_activation_rx - .wait_for(|active| *active) - .await - .is_err() - { - return; - } - let first_tick = tokio::time::Instant::now() + heartbeat_interval; - let mut interval = tokio::time::interval_at(first_tick, heartbeat_interval); + let mut interval = tokio::time::interval(heartbeat_interval); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Give startup/WAL replay one full interval before the first + // carrier. A missed/full notification is harmless: a later tick + // retries the still-open local slot. + interval.tick().await; loop { interval.tick().await; let Some(heartbeat_tx) = heartbeat_tx.upgrade() else { @@ -1561,86 +757,23 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }); } - // One persistent, generation-tagged deadline task replaces per-attempt - // sleeps. Repeated normal creation requests only replace the desired - // deadline in the watch channel; a stale queued wake is harmless. - let normal_carrier_tx = message_tx.downgrade(); - tokio::spawn(async move { - loop { - if normal_carrier_deadline_rx.changed().await.is_err() { - return; - } - let Some(mut target) = *normal_carrier_deadline_rx.borrow_and_update() else { - continue; - }; - loop { - tokio::select! { - _ = tokio::time::sleep_until(tokio::time::Instant::from_std(target.deadline)) => { - if *normal_carrier_deadline_rx.borrow() != Some(target) { - break; - } - let Some(sender) = normal_carrier_tx.upgrade() else { - return; - }; - if sender - .send(ShadowServiceMessageV1::NormalCarrierDeadline { - generation: target.generation, - }) - .await - .is_err() - { - return; - } - break; - } - changed = normal_carrier_deadline_rx.changed() => { - if changed.is_err() { - return; - } - match *normal_carrier_deadline_rx.borrow_and_update() { - Some(replacement) => target = replacement, - None => break, - } - } - } - } - } - }); - spawn_consensus_timeout_deadline_task(consensus_timeout_deadline_rx, message_tx.downgrade()); let startup_events = event_tx.clone(); let actor_desired_topology = Arc::clone(&desired_topology); - let actor_desired_local_applications = Arc::clone(&desired_local_applications); - let actor_desired_carrier_sync_responses = Arc::clone(&desired_carrier_sync_responses); - let actor_desired_verified_application_payloads = - Arc::clone(&desired_verified_application_payloads); let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); let actor_desired_available_applications = Arc::clone(&desired_available_applications); let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); let actor_retry_notification_pending = Arc::clone(&retry_notification_pending); let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); - let actor_committee = committee.clone(); let task = tokio::spawn(async move { let opened = tokio::task::spawn_blocking(move || { - if mode.is_autonomous() { - StarfishRbcDagShadowV1::open_authoritative_with_wal_sync_policy( - path, - committee, - own_authority, - context, - authorizer, - wal_sync_policy, - recovery_cursor, - ) - } else { - StarfishRbcDagShadowV1::open_with_wal_sync_policy( - path, - committee, - own_authority, - context, - authorizer, - wal_sync_policy, - ) - } + StarfishRbcDagShadowV1::open_with_wal_sync_policy( + path, + committee, + own_authority, + context, + authorizer, + wal_sync_policy, + ) }) .await; let (core, open_report) = match opened { @@ -1815,8 +948,11 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }; let recovered_shadow_deliveries = reported_shadow_deliveries.clone(); - let recovered_application_headers = match core.delivered_application_headers() { - Ok(headers) => headers, + let reported_application_deliveries = match core.delivered_application_headers() { + Ok(headers) => headers + .into_iter() + .map(|(_, header)| header.reference()) + .collect(), Err(error) => { let _ = startup_events .send(ShadowServiceEventV1::Rejected { @@ -1827,57 +963,21 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( return; } }; - let reported_application_deliveries = recovered_application_headers - .iter() - .map(|(_, header)| header.reference()) - .collect(); - let mut authorized_applications = BTreeMap::new(); - for (carrier, header) in recovered_application_headers - .into_iter() - .rev() - .take(SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1) - { - authorized_applications.insert( - header.reference(), - AuthorizedApplicationStateV1 { - carrier, - header, - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - verified_payload: None, - observed_payload: None, - observed_payload_at: None, - holders: BTreeSet::new(), - request_last_attempt: BTreeMap::new(), - emitted_header: false, - emitted_observed_payload: false, - }, - ); - } let reported_shadow_delivery_slots = reported_shadow_deliveries .iter() .map(delivery_slot) .collect(); let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); let sync_round = core.local_carrier_round(); - let consensus_pacemaker = ConsensusPacemakerV1::new(core.next_local_consensus_round()); let state = ShadowServiceStateV1 { core, - committee: actor_committee, mode, - clock_active: false, - clock_activation_tx, wal_sync_policy, - metrics, own_authority, committee_size, events: event_tx, connected: BTreeSet::new(), - catch_up_hint_high_water: BTreeMap::new(), - far_future_hint_high_water: BTreeMap::new(), desired_topology: actor_desired_topology, - desired_local_applications: actor_desired_local_applications, - desired_carrier_sync_responses: actor_desired_carrier_sync_responses, - desired_verified_application_payloads: actor_desired_verified_application_payloads, desired_direct_deliveries: actor_desired_direct_deliveries, desired_available_applications: actor_desired_available_applications, observed_topology: BTreeMap::new(), @@ -1889,33 +989,10 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), sync_last_served: BTreeMap::new(), - authorized_applications, - quarantined_application_payloads: BTreeMap::new(), - payload_last_served: BTreeMap::new(), sync_round, sync_round_opened_at: Instant::now(), sync_catch_up: false, - sync_catch_up_limit_future: false, - sync_catch_up_target: None, - #[cfg(test)] - sync_max_outstanding: 0, - #[cfg(test)] - sync_max_desired_responses: 0, - awaiting_application_submission: false, - consensus_pacemaker, - normal_carrier_min_spacing: mode - .heartbeat_interval() - .and_then(|interval| interval.checked_div(SHADOW_NORMAL_CARRIER_SPACING_DIVISOR_V1)) - .unwrap_or_default() - .max(SHADOW_NORMAL_CARRIER_MIN_SPACING_V1), - normal_carrier_next_allowed_at: None, - normal_carrier_requested: false, - normal_carrier_generation: 0, - normal_carrier_deadline: None, - normal_carrier_deadline_tx, - consensus_timeout_generation: 0, - consensus_timeout_deadline: None, - consensus_timeout_deadline_tx, + sync_used_in_open_round: false, retry_notification_pending: actor_retry_notification_pending, heartbeat_notification_pending: actor_heartbeat_notification_pending, direct_deliveries: BTreeSet::new(), @@ -1929,7 +1006,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( fatal: false, }; if let Err(error) = tokio::task::spawn_blocking(move || { - run_shadow_service(state, message_rx, open_report, clock_starts_active); + run_shadow_service(state, message_rx, open_report); }) .await { @@ -1947,13 +1024,9 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( max_sidecar_size, own_authority, committee_size, - #[cfg(test)] input_capacity, mode, desired_topology, - desired_local_applications, - desired_carrier_sync_responses, - desired_verified_application_payloads, desired_direct_deliveries, desired_available_applications, invalidated_by_overload, @@ -2027,126 +1100,15 @@ impl ShadowComparisonBacklogV1 { } } -#[derive(Clone, Debug)] -struct AuthorizedApplicationStateV1 { - carrier: BlockReference, - header: RbcCanonicalHeader, - authorization_basis: ShadowApplicationAuthorizationBasisV1, - verified_payload: Option>, - observed_payload: Option>, - observed_payload_at: Option, - holders: BTreeSet, - request_last_attempt: BTreeMap, - emitted_header: bool, - emitted_observed_payload: bool, -} - -#[derive(Clone, Debug)] -struct QuarantinedApplicationPayloadV1 { - application: BlockReference, - holder: AuthorityIndex, - payload: Option>, -} - -/// Per-requester exact-slot replay limiter. A lagging honest peer may consume -/// a bounded set of distinct rounds faster than the healthy carrier clock, -/// independent of response/request ordering. Replays of a round already -/// served in the current interval and a seventeenth distinct round are -/// throttled. -#[derive(Clone, Debug)] -struct CarrierSyncServeWindowV1 { - started_at: Instant, - served_rounds: BTreeSet, -} - -impl CarrierSyncServeWindowV1 { - fn first(round: RoundNumber, now: Instant) -> Self { - let mut served_rounds = BTreeSet::new(); - served_rounds.insert(round); - Self { - started_at: now, - served_rounds, - } - } - - fn permits(&mut self, round: RoundNumber, now: Instant) -> bool { - if now.saturating_duration_since(self.started_at) >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 { - self.started_at = now; - self.served_rounds.clear(); - } - if self.served_rounds.contains(&round) - || self.served_rounds.len() >= SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1 - { - return false; - } - self.served_rounds.insert(round) - } -} - -fn carrier_sync_pipeline_depth(committee_size: usize) -> usize { - let remote_authors = committee_size.saturating_sub(1); - if remote_authors == 0 { - return 0; - } - SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1 - .min(SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 / remote_authors) -} - -/// Produce the bounded round-major repair window. `target == None` is the -/// healthy/moderate single-round path; a catch-up target opens only enough -/// consecutive rounds to keep all `n - 1` authors inside the global credit. -fn carrier_sync_pipeline_slots( - current_round: RoundNumber, - target: Option, - committee_size: usize, - connected: &BTreeSet, -) -> Vec { - let depth = carrier_sync_pipeline_depth(committee_size); - if depth == 0 { - return Vec::new(); - } - let depth_delta = RoundNumber::try_from(depth.saturating_sub(1)).unwrap_or(RoundNumber::MAX); - let last_round = current_round - .saturating_add(depth_delta) - .min(target.unwrap_or(current_round)); - if last_round < current_round { - return Vec::new(); - } - (current_round..=last_round) - .flat_map(|round| connected.iter().copied().map(move |author| (round, author))) - .take(SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1) - .collect() -} - struct ShadowServiceStateV1 { core: StarfishRbcDagShadowV1, - committee: RbcDagCommitteeContextV1, mode: ShadowServiceModeV1, - /// One-way coordinated-start latch for the autonomous protocol clock. - /// Mirror mode never consults this flag. - clock_active: bool, - clock_activation_tx: watch::Sender, wal_sync_policy: ShadowWalSyncPolicyV1, - metrics: Option>, own_authority: AuthorityIndex, committee_size: usize, events: mpsc::Sender, connected: BTreeSet, - /// Distinct transport-authenticated peers that exposed a carrier beyond - /// the local admission/retention horizon. Validity stake is required - /// before this performance-only hint can enable aggressive catch-up, so a - /// Byzantine minority cannot force the node into repair mode. - catch_up_hint_high_water: BTreeMap, - /// Subset of catch-up hints that were outside the authenticated 64-round - /// retention horizon. Only validity stake in this set disables proactive - /// future buffering; ordinary small skew keeps the useful retained tail. - far_future_hint_high_water: BTreeMap, desired_topology: Arc>>, - desired_local_applications: Arc>>, - desired_carrier_sync_responses: - Arc>>, - desired_verified_application_payloads: - Arc>>>, desired_direct_deliveries: Arc>>, desired_available_applications: Arc>>, observed_topology: BTreeMap, @@ -2156,35 +1118,12 @@ struct ShadowServiceStateV1 { pending_data_availability: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, - sync_last_attempt: BTreeMap, - sync_last_served: BTreeMap, - authorized_applications: BTreeMap, - quarantined_application_payloads: BTreeMap, - payload_last_served: BTreeMap, + sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, + sync_last_served: BTreeMap, sync_round: RoundNumber, sync_round_opened_at: Instant, sync_catch_up: bool, - sync_catch_up_limit_future: bool, - sync_catch_up_target: Option, - #[cfg(test)] - sync_max_outstanding: usize, - #[cfg(test)] - sync_max_desired_responses: usize, - /// A successful application-carrier assignment just released the core's - /// one-outstanding producer gate. Give that exact producer one actor turn - /// to submit its successor before C3 spends the next physical slot on a - /// control heartbeat. Maintenance/heartbeat messages bound the wait. - awaiting_application_submission: bool, - consensus_pacemaker: ConsensusPacemakerV1, - normal_carrier_min_spacing: Duration, - normal_carrier_next_allowed_at: Option, - normal_carrier_requested: bool, - normal_carrier_generation: u64, - normal_carrier_deadline: Option, - normal_carrier_deadline_tx: watch::Sender>, - consensus_timeout_generation: u64, - consensus_timeout_deadline: Option, - consensus_timeout_deadline_tx: watch::Sender>, + sync_used_in_open_round: bool, retry_notification_pending: Arc, heartbeat_notification_pending: Arc, direct_deliveries: BTreeSet, @@ -2203,51 +1142,6 @@ impl ShadowServiceStateV1 { let _ = self.events.blocking_send(event); } - fn activate_clock(&mut self) { - if !self.mode.is_autonomous() || self.clock_active { - return; - } - self.clock_active = true; - let now = Instant::now(); - self.sync_round = self.core.local_carrier_round(); - self.sync_round_opened_at = now; - self.cancel_consensus_timeout_deadline(); - self.consensus_pacemaker = - ConsensusPacemakerV1::new(self.core.next_local_consensus_round()); - self.awaiting_application_submission = false; - self.heartbeat_notification_pending - .store(false, Ordering::Release); - - // Queue the ordered activation observation before releasing the timer - // task. Even if the event channel is temporarily full, no heartbeat - // can be generated ahead of `ClockActivated`. - self.emit(ShadowServiceEventV1::ClockActivated); - self.clock_activation_tx.send_replace(true); - self.emit_clock_state(); - } - - fn emit_recovered_authorized_applications(&mut self) { - let applications = self - .authorized_applications - .keys() - .copied() - .collect::>(); - for application in applications { - self.emit_authorized_application_if_new(application); - } - } - - fn emit_autonomous_recovery_and_ready(&mut self, open_report: &ShadowOpenReportV1) { - self.emit_recovered_authorized_applications(); - for delta in open_report.recovered_committed_frontiers() { - self.emit(ShadowServiceEventV1::FrontierCommitted(delta.clone())); - } - self.process_effects(open_report.recovery_effects().to_vec()); - self.emit(ShadowServiceEventV1::Ready { - autonomous_clock: true, - }); - } - fn reject(&self, peer: Option, error: impl fmt::Display) { self.emit(ShadowServiceEventV1::Rejected { peer, @@ -2271,7 +1165,6 @@ impl ShadowServiceStateV1 { if !self.mode.is_autonomous() { return; } - self.record_pipeline_state(); self.emit(ShadowServiceEventV1::ClockState { open_round: self.core.local_carrier_round(), phase_backlog: self.core.pending_phase_backlog_len(), @@ -2281,22 +1174,6 @@ impl ShadowServiceStateV1 { }); } - fn record_pipeline_state(&self) { - let Some(metrics) = &self.metrics else { - return; - }; - let projection = self.core.projection_runtime_snapshot(); - metrics.set_starfish_rbc_dag_pipeline_state( - self.pending_local.len(), - projection.pending_candidates, - projection.highest_projected_round, - projection.next_undecided_round, - projection.next_undecided_projected_stake, - projection.last_committed_round, - projection.hol_reason.metric_label(), - ); - } - fn validate_peer(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { if peer as usize >= self.committee_size { return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); @@ -2307,512 +1184,6 @@ impl ShadowServiceStateV1 { Ok(()) } - fn decode_application_carrier( - &self, - canonical_carrier: &[u8], - ) -> Result, ShadowErrorV1> { - let candidate = CandidateCarrierV1::decode_wire_with_committee( - canonical_carrier, - &self.committee, - None, - ) - .map_err(ShadowErrorV1::Carrier)?; - Ok(candidate - .header() - .application_header() - .cloned() - .map(|header| (candidate.reference(), header))) - } - - fn make_application_state_room(&mut self, application: BlockReference) { - if self.authorized_applications.contains_key(&application) - || self.authorized_applications.len() < SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 - { - return; - } - if let Some((evicted, _)) = self.authorized_applications.pop_first() { - self.quarantined_application_payloads - .retain(|_, retained| retained.application != evicted); - } - } - - fn emit_authorized_application_if_new(&mut self, application: BlockReference) { - let event = self - .authorized_applications - .get_mut(&application) - .and_then(|state| { - let should_emit = !state.emitted_header - || (state.observed_payload.is_some() && !state.emitted_observed_payload); - should_emit.then(|| { - state.emitted_header = true; - state.emitted_observed_payload |= state.observed_payload.is_some(); - ShadowServiceEventV1::AuthorizedApplicationObserved { - carrier: state.carrier, - header: state.header.clone(), - payload: state - .observed_payload - .clone() - .or_else(|| state.verified_payload.clone()), - authorization_basis: state.authorization_basis, - } - }) - }); - if let Some(event) = event { - self.emit(event); - } - } - - fn authorize_application( - &mut self, - carrier: BlockReference, - header: RbcCanonicalHeader, - payload: Option>, - payload_is_verified: bool, - holder: Option, - authorization_basis: ShadowApplicationAuthorizationBasisV1, - ) -> Result<(), ShadowServiceErrorV1> { - let application = header.reference(); - self.make_application_state_room(application); - let state = self - .authorized_applications - .entry(application) - .or_insert_with(|| AuthorizedApplicationStateV1 { - carrier, - header: header.clone(), - authorization_basis, - verified_payload: None, - observed_payload: None, - observed_payload_at: None, - holders: BTreeSet::new(), - request_last_attempt: BTreeMap::new(), - emitted_header: false, - emitted_observed_payload: false, - }); - if state.header != header { - return Err(ShadowServiceErrorV1::ConflictingApplicationPayload( - application, - )); - } - if let Some(holder) = holder { - state.holders.insert(holder); - } - if payload_is_verified { - merge_application_payload(&mut state.verified_payload, payload, application)?; - } else { - let observed = payload.is_some(); - merge_application_payload(&mut state.observed_payload, payload, application)?; - if observed { - state.observed_payload_at = Some(Instant::now()); - } - } - self.emit_authorized_application_if_new(application); - self.flush_application_payload_requests(); - Ok(()) - } - - fn quarantine_application( - &mut self, - carrier: BlockReference, - application: BlockReference, - holder: AuthorityIndex, - payload: Option>, - ) -> Result<(), ShadowServiceErrorV1> { - if !self.quarantined_application_payloads.contains_key(&carrier) - && self.quarantined_application_payloads.len() >= SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 - { - self.quarantined_application_payloads.pop_first(); - } - let retained = self - .quarantined_application_payloads - .entry(carrier) - .or_insert(QuarantinedApplicationPayloadV1 { - application, - holder, - payload: None, - }); - if retained.application != application { - return Err(ShadowServiceErrorV1::ConflictingApplicationPayload( - application, - )); - } - merge_application_payload(&mut retained.payload, payload, application) - } - - fn observe_carrier_application( - &mut self, - peer: AuthorityIndex, - canonical_carrier: &[u8], - payload: Option>, - disposition: ShadowIngressDispositionV1, - ) -> Result<(), ShadowServiceErrorV1> { - if disposition == ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer { - return Ok(()); - } - let Some((carrier, header)) = self.decode_application_carrier(canonical_carrier)? else { - return Ok(()); - }; - let application = header.reference(); - let (observed_payload, payload_error) = match payload { - Some(payload) => match validate_application_payload_size(&payload) { - Ok(()) => (Some(payload), None), - Err(error) => (None, Some(error)), - }, - None => (None, None), - }; - match disposition { - ShadowIngressDispositionV1::Authenticated => self.authorize_application( - carrier, - header, - observed_payload, - false, - Some(peer), - ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, - )?, - ShadowIngressDispositionV1::CandidateRetained => { - self.quarantine_application(carrier, application, peer, observed_payload)? - } - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { - if self - .authorized_applications - .get(&application) - .is_some_and(|state| state.carrier == carrier) - { - self.authorize_application( - carrier, - header, - observed_payload, - false, - Some(peer), - ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, - )?; - } else if self.quarantined_application_payloads.contains_key(&carrier) { - self.quarantine_application(carrier, application, peer, observed_payload)?; - } - } - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => {} - } - if let Some(error) = payload_error { - return Err(error); - } - Ok(()) - } - - fn authorize_delivered_application( - &mut self, - carrier: BlockReference, - header: RbcCanonicalHeader, - ) -> Result<(), ShadowServiceErrorV1> { - let retained = self.quarantined_application_payloads.remove(&carrier); - let holder = retained.as_ref().map(|retained| retained.holder); - let payload = retained.and_then(|retained| retained.payload); - self.authorize_application( - carrier, - header, - payload, - false, - holder, - ShadowApplicationAuthorizationBasisV1::Delivered, - ) - } - - fn flush_application_payload_requests(&mut self) { - let now = Instant::now(); - let connected = &self.connected; - let mut requests = Vec::new(); - for (application, state) in &self.authorized_applications { - // The default commitment is the canonical proof of an empty - // application payload. DagState marks such a header available as - // soon as it is materialized, so there are no payload bytes to - // recover and no peer can produce a meaningful response. - if state.header.transactions_commitment() == TransactionsCommitment::default() - || state.verified_payload.is_some() - || state.observed_payload_at.is_some_and(|observed| { - now.saturating_duration_since(observed) - < SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 - }) - || state.request_last_attempt.values().any(|last| { - now.saturating_duration_since(*last) - < SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 - }) - { - continue; - } - let peer = state - .holders - .iter() - .copied() - .filter(|holder| connected.contains(holder)) - .chain(connected.iter().copied()) - .find(|holder| { - state.request_last_attempt.get(holder).is_none_or(|last| { - now.saturating_duration_since(*last) - >= SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 - }) - }); - if let Some(peer) = peer { - requests.push((*application, peer)); - } - } - for (application, peer) in requests { - let Some(state) = self.authorized_applications.get_mut(&application) else { - continue; - }; - state.observed_payload = None; - state.observed_payload_at = None; - state.emitted_observed_payload = false; - state.request_last_attempt.clear(); - state.request_last_attempt.insert(peer, now); - self.emit(ShadowServiceEventV1::Network { - recipient: peer, - message: NetworkMessage::RbcDagApplicationPayloadRequest(application), - }); - } - } - - fn handle_application_payload_request( - &mut self, - peer: AuthorityIndex, - application: BlockReference, - ) { - let now = Instant::now(); - if self - .payload_last_served - .get(&peer) - .is_some_and(|(_, last)| { - now.saturating_duration_since(*last) < SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1 - }) - { - self.emit(ShadowServiceEventV1::Input { - kind: "application_payload_request", - outcome: "rate_limited", - }); - return; - } - let Some(payload) = self - .authorized_applications - .get(&application) - .and_then(|state| state.verified_payload.clone()) - else { - self.emit(ShadowServiceEventV1::Input { - kind: "application_payload_request", - outcome: "not_found", - }); - return; - }; - self.payload_last_served.insert(peer, (application, now)); - self.emit(ShadowServiceEventV1::Network { - recipient: peer, - message: NetworkMessage::RbcDagApplicationPayloadResponse( - RbcDagApplicationPayloadResponse { - application, - transaction_data: payload, - }, - ), - }); - } - - fn handle_application_payload_response( - &mut self, - peer: AuthorityIndex, - response: RbcDagApplicationPayloadResponse, - ) -> Result<(), ShadowServiceErrorV1> { - let application = response.application; - let Some(state) = self.authorized_applications.get(&application) else { - // Verification and network delivery run outside the actor. A - // solicited response can therefore arrive after the bounded - // application window evicted its state. Core either already saw - // the payload or later recovery will reauthorize the header; the - // delayed bytes must not invalidate an otherwise healthy run. - self.emit(ShadowServiceEventV1::Input { - kind: "application_payload_response", - outcome: "stale_ignored", - }); - return Ok(()); - }; - if state.verified_payload.is_some() { - return Ok(()); - } - if !state.request_last_attempt.contains_key(&peer) { - return Err( - ShadowServiceErrorV1::ApplicationPayloadResponseFromUnexpectedPeer { - peer, - application, - }, - ); - } - validate_application_payload_size(&response.transaction_data)?; - let state = self - .authorized_applications - .get_mut(&application) - .expect("authorized application cannot disappear during validation"); - let duplicate = application_payloads_equal( - state.observed_payload.as_deref(), - Some(response.transaction_data.as_ref()), - ); - merge_application_payload( - &mut state.observed_payload, - Some(response.transaction_data), - application, - )?; - state.observed_payload_at = Some(Instant::now()); - if !duplicate { - state.emitted_observed_payload = false; - } - self.emit_authorized_application_if_new(application); - Ok(()) - } - - fn handle_verified_application_payload( - &mut self, - application: BlockReference, - payload: Arc, - ) -> Result<(), ShadowServiceErrorV1> { - let Some(state) = self.authorized_applications.get_mut(&application) else { - // Core verification/materialization is deliberately offloaded - // from this actor. Its completion may therefore trail the bounded - // authorized-application retention window. The payload already - // reached Core; it must not turn normal bounded eviction into a - // protocol rejection or invalidate the benchmark. - self.emit(ShadowServiceEventV1::Input { - kind: "verified_application_payload", - outcome: "stale_ignored", - }); - return Ok(()); - }; - merge_application_payload(&mut state.verified_payload, Some(payload), application)?; - state.observed_payload = None; - state.observed_payload_at = None; - state.request_last_attempt.clear(); - self.emit(ShadowServiceEventV1::Input { - kind: "verified_application_payload", - outcome: "cached", - }); - Ok(()) - } - - fn reconcile_local_applications(&mut self) { - let desired = std::mem::take(&mut *self.desired_local_applications.lock()); - if !desired.is_empty() { - self.awaiting_application_submission = false; - } - for local in desired.into_values() { - self.enqueue_local(local); - } - } - - fn reconcile_verified_application_payloads(&mut self) { - let desired = std::mem::take(&mut *self.desired_verified_application_payloads.lock()); - for (application, payload) in desired { - if let Err(error) = self.handle_verified_application_payload(application, payload) { - self.reject(None, error); - } - } - } - - fn validity_backed_high_water( - &self, - hints: &BTreeMap, - ) -> Option { - let mut candidate_rounds = hints.values().copied().collect::>(); - candidate_rounds.sort_unstable(); - candidate_rounds.dedup(); - candidate_rounds.into_iter().rev().find(|round| { - let stake = hints - .iter() - .filter(|(_, high_water)| **high_water >= *round) - .filter_map(|(authority, _)| self.committee.committee().get_stake(*authority)) - .fold(0, u64::saturating_add); - self.committee.committee().is_valid(stake) - }) - } - - fn observe_catch_up_hint( - &mut self, - peer: AuthorityIndex, - round: RoundNumber, - outside_retention: bool, - ) { - if !self.mode.is_autonomous() { - return; - } - self.catch_up_hint_high_water - .entry(peer) - .and_modify(|high_water| *high_water = (*high_water).max(round)) - .or_insert(round); - if outside_retention { - self.far_future_hint_high_water - .entry(peer) - .and_modify(|high_water| *high_water = (*high_water).max(round)) - .or_insert(round); - } - let high_water = self.validity_backed_high_water(&self.catch_up_hint_high_water); - let far_high_water = self.validity_backed_high_water(&self.far_future_hint_high_water); - // Ordinary authenticated lookahead inside the 64-round retention - // window is the healthy proactive pipeline, not evidence that exact - // catch-up is needed. Starting a 64-slot repair episode for those - // observations made healthy nodes continuously request data they had - // already buffered. Activate pipelined catch-up only after validity - // stake reports a round outside normal retention. - let activated = !self.sync_catch_up && far_high_water.is_some(); - let limited = !self.sync_catch_up_limit_future && far_high_water.is_some(); - if far_high_water.is_some() { - let high_water = high_water.expect("far-future hints are also catch-up hints"); - self.sync_catch_up_target = Some( - self.sync_catch_up_target - .map_or(high_water, |target| target.max(high_water)), - ); - self.sync_catch_up = true; - } - if activated { - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_catch_up", - outcome: "validity_outside_retention_hints", - }); - } - if limited { - self.sync_catch_up = true; - self.sync_catch_up_limit_future = true; - } - } - - fn observe_ingress_catch_up_hint( - &mut self, - peer: AuthorityIndex, - canonical_carrier: &[u8], - disposition: ShadowIngressDispositionV1, - open_round_before: RoundNumber, - ) { - let Ok((_, hint_round, _)) = self.core.candidate_slot(canonical_carrier) else { - return; - }; - // Count deltas are not a sound lookahead signal: the same ingress can - // advance the clock, promote older buffered slots, and leave the total - // flat. Classify against the pre-ingress open round instead. - let authenticated_future = disposition == ShadowIngressDispositionV1::Authenticated - && hint_round > open_round_before.saturating_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1); - let future_ignored = disposition == ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer; - if !future_ignored && !authenticated_future { - return; - } - self.observe_catch_up_hint( - peer, - hint_round, - hint_round > open_round_before.saturating_add(EXECUTABLE_MODEL_BUFFER_WINDOW_V1), - ); - } - - /// Highest physical carrier round reported by validity stake while still - /// retained by the normal future window. A carrier at round `target` - /// proves that a live producer may already have opened `target + 1`, so - /// local catch-up is complete only after our open round passes it. - fn retained_future_target(&self) -> Option { - if !self.mode.is_autonomous() || self.sync_catch_up { - return None; - } - let target = self.validity_backed_high_water(&self.catch_up_hint_high_water)?; - (self.core.local_carrier_round() <= target).then_some(target) - } - fn reconcile_topology(&mut self) { let desired = self.desired_topology.lock().clone(); let mut newly_connected = Vec::new(); @@ -2823,20 +1194,14 @@ impl ShadowServiceStateV1 { self.recovery_last_attempt .retain(|(_, holder), _| holder != peer); self.sync_last_attempt - .retain(|(_, author), _| author != peer); + .retain(|(author, _), _| author != peer); self.sync_last_served .retain(|requester, _| requester != peer); - self.payload_last_served.remove(peer); - for application in self.authorized_applications.values_mut() { - application.request_last_attempt.remove(peer); - } if state.0 { self.connected.insert(*peer); newly_connected.push(*peer); } else { self.connected.remove(peer); - self.catch_up_hint_high_water.remove(peer); - self.far_future_hint_high_water.remove(peer); } } self.observed_topology = desired; @@ -2845,10 +1210,7 @@ impl ShadowServiceStateV1 { // Autonomous history is synchronized one exact slot at a // time. Replaying the entire retained run on every reconnect // would create an unbounded burst as heartbeats accumulate. - // The first physical carrier may have been fixed before the - // network connection existed, so round one needs the same - // immediate exact-slot repair as every later reconnect. - self.flush_carrier_sync_requests(true); + self.flush_carrier_sync_requests(self.core.local_carrier_round() > 1); } else { let retransmissions = self.core.retransmissions(); for peer in newly_connected { @@ -2858,7 +1220,6 @@ impl ShadowServiceStateV1 { } } self.flush_recovery_requests(); - self.flush_application_payload_requests(); } } @@ -2912,42 +1273,20 @@ impl ShadowServiceStateV1 { } fn broadcast(&self, envelope: &ShadowOutboundEnvelopeV1) { - self.broadcast_with_application_payload(envelope, None); - } - - fn broadcast_with_application_payload( - &self, - envelope: &ShadowOutboundEnvelopeV1, - application_payload: Option>, - ) { for recipient in 0..self.committee_size { let recipient = recipient as AuthorityIndex; if recipient != self.own_authority { - self.send_envelope_with_application_payload( - recipient, - envelope, - application_payload.clone(), - ); + self.send_envelope(recipient, envelope); } } } fn send_envelope(&self, recipient: AuthorityIndex, envelope: &ShadowOutboundEnvelopeV1) { - self.send_envelope_with_application_payload(recipient, envelope, None); - } - - fn send_envelope_with_application_payload( - &self, - recipient: AuthorityIndex, - envelope: &ShadowOutboundEnvelopeV1, - application_payload: Option>, - ) { self.emit(ShadowServiceEventV1::Network { recipient, message: NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { canonical_carrier: envelope.canonical_carrier_wire().to_vec(), authentication_sidecar: envelope.authentication_sidecar().to_vec(), - application_payload, }), }); } @@ -2966,17 +1305,21 @@ impl ShadowServiceStateV1 { } fn process_effects(&mut self, effects: Vec) { + let carrier_round_advanced = effects + .iter() + .any(|effect| matches!(effect, ModelEffect::CarrierRoundAdvanced(_))); let newly_delivered = effects .iter() .filter_map(|effect| match effect { - ModelEffect::Delivered(reference) | ModelEffect::DeliveryPromised(reference) => { - Some(*reference) - } + ModelEffect::Delivered(reference) => Some(*reference), ModelEffect::NeedCarrier { .. } | ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => None, }) .collect::>(); + if carrier_round_advanced { + self.sync_catch_up = std::mem::take(&mut self.sync_used_in_open_round); + } for effect in effects { match effect { ModelEffect::NeedCarrier { target, holders } => { @@ -2987,7 +1330,7 @@ impl ShadowServiceStateV1 { }), ); } - ModelEffect::Delivered(reference) | ModelEffect::DeliveryPromised(reference) => { + ModelEffect::Delivered(reference) => { self.pending_recovery.remove(&reference); } ModelEffect::PrefixAdvanced { .. } => {} @@ -3003,7 +1346,6 @@ impl ShadowServiceStateV1 { self.report_new_shadow_deliveries(&newly_delivered); self.report_projection_progress(); self.flush_carrier_sync_requests(false); - self.refresh_consensus_pacemaker(); self.emit_clock_state(); } @@ -3056,211 +1398,15 @@ impl ShadowServiceStateV1 { self.emit(ShadowServiceEventV1::LeaderDecided(decision)); } for delta in self.core.drain_committed_frontiers() { - if let Some(metrics) = &self.metrics { - let (total_ns, samples, max_ns) = latency_since_header_creation( - delta.applications.iter(), - current_timestamp_ns(), - ); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_FRONTIER_GENERATED, - total_ns, - samples, - max_ns, - ); - metrics.starfish_rbc_dag_frontier_generated(); - } self.emit(ShadowServiceEventV1::FrontierCommitted(delta)); } } - fn refresh_consensus_pacemaker(&mut self) { - let _ = self.consensus_fallback_allowed_at(Instant::now()); - } - - fn consensus_fallback_allowed_at(&mut self, now: Instant) -> bool { - if !self.clock_active { - return false; - } - let Some(leader_timeout) = self.mode.heartbeat_interval() else { - return false; - }; - let slot = self.core.next_local_consensus_round(); - let a1_ready = slot == 1 - || self - .core - .has_projected_consensus_quorum(slot.saturating_sub(1)); - let c3_ready = self.core.has_projected_consensus_quorum(slot); - if self.consensus_pacemaker.slot != slot { - self.cancel_consensus_timeout_deadline(); - } - let fallback_allowed = self.consensus_pacemaker.fallback_allowed( - slot, - a1_ready, - c3_ready, - leader_timeout, - now, - ); - if fallback_allowed { - // C3 or an elapsed C2 already authorizes this exact slot. The - // normal-carrier permit may defer creation, but it will re-check - // the same monotonic evidence and does not need another timeout. - self.cancel_consensus_timeout_deadline(); - } else if let Some(armed_at) = self.consensus_pacemaker.c2_armed_at { - let deadline = armed_at.checked_add(leader_timeout).unwrap_or(now); - self.schedule_consensus_timeout_deadline(slot, deadline); - } else { - self.cancel_consensus_timeout_deadline(); - } - fallback_allowed - } - - fn cancel_consensus_timeout_deadline(&mut self) { - if self.consensus_timeout_deadline.take().is_some() { - self.consensus_timeout_generation = self.consensus_timeout_generation.wrapping_add(1); - self.consensus_timeout_deadline_tx.send_replace(None); - } - } - - fn schedule_consensus_timeout_deadline(&mut self, slot: RoundNumber, deadline: Instant) { - if self - .consensus_timeout_deadline - .is_some_and(|scheduled| scheduled.slot == slot && scheduled.deadline == deadline) - { - return; - } - self.consensus_timeout_generation = self.consensus_timeout_generation.wrapping_add(1); - let scheduled = ConsensusTimeoutDeadlineV1 { - generation: self.consensus_timeout_generation, - slot, - deadline, - }; - self.consensus_timeout_deadline = Some(scheduled); - self.consensus_timeout_deadline_tx - .send_replace(Some(scheduled)); - } - - fn observe_consensus_timeout_deadline(&mut self, generation: u64, slot: RoundNumber) { - let Some(scheduled) = self.consensus_timeout_deadline else { - return; - }; - if scheduled.generation != generation || scheduled.slot != slot { - return; - } - if Instant::now() < scheduled.deadline { - // Fail closed if an internal wake is ever observed early. Notify - // the persistent task so it continues waiting for the exact - // actor-owned deadline. - self.consensus_timeout_deadline_tx - .send_replace(Some(scheduled)); - return; - } - self.consensus_timeout_deadline = None; - self.consensus_timeout_deadline_tx.send_replace(None); - if self.core.next_local_consensus_round() != slot - || !self.consensus_pacemaker.observe_timeout(slot) - { - return; - } - self.awaiting_application_submission = false; - } - - fn cancel_normal_carrier_deadline(&mut self) { - if self.normal_carrier_deadline.take().is_some() { - self.normal_carrier_generation = self.normal_carrier_generation.wrapping_add(1); - self.normal_carrier_deadline_tx.send_replace(None); - } - } - - fn record_carrier_created(&mut self, now: Instant) { - self.normal_carrier_requested = false; - self.cancel_normal_carrier_deadline(); - self.normal_carrier_next_allowed_at = Some( - now.checked_add(self.normal_carrier_min_spacing) - .unwrap_or(now), - ); - } - - fn schedule_normal_carrier_deadline(&mut self, deadline: Instant) { - self.normal_carrier_requested = true; - if self - .normal_carrier_deadline - .is_some_and(|scheduled| scheduled.deadline == deadline) - { - return; - } - self.normal_carrier_generation = self.normal_carrier_generation.wrapping_add(1); - let scheduled = NormalCarrierDeadlineV1 { - generation: self.normal_carrier_generation, - deadline, - }; - self.normal_carrier_deadline = Some(scheduled); - self.normal_carrier_deadline_tx - .send_replace(Some(scheduled)); - } - - /// Request one normally paced carrier. All application, phase, heartbeat, - /// and C1/C2/C3 creation paths share this permit. Only validity-backed - /// exact/retained repair may call the unpaced primitive directly. - fn try_create_autonomous_carrier(&mut self) { - if !self.mode.is_autonomous() || !self.clock_active { + fn try_create_autonomous_carrier(&mut self, allow_no_vote: bool) { + if !self.mode.is_autonomous() || !self.core.can_create_carrier() { self.emit_clock_state(); return; } - let now = Instant::now(); - if let Some(deadline) = self.normal_carrier_next_allowed_at { - if now < deadline { - self.schedule_normal_carrier_deadline(deadline); - return; - } - } - self.normal_carrier_requested = false; - self.cancel_normal_carrier_deadline(); - let _ = self.try_create_autonomous_carrier_now(); - } - - fn observe_normal_carrier_deadline(&mut self, generation: u64) { - let Some(scheduled) = self.normal_carrier_deadline else { - return; - }; - if scheduled.generation != generation { - return; - } - self.normal_carrier_deadline = None; - self.normal_carrier_deadline_tx.send_replace(None); - if self.normal_carrier_requested { - self.try_create_autonomous_carrier(); - } - } - - /// C2 and C3 are creation triggers, not merely permission for the next - /// fixed heartbeat. Attempt at most one carrier outside `process_effects` - /// so reducing a locally created carrier cannot recurse into creation. - fn drive_consensus_fallback(&mut self) { - if !self.clock_active { - return; - } - let fallback_allowed = self.consensus_fallback_allowed_at(Instant::now()); - if !fallback_allowed - || !self.mode.is_autonomous() - || !self.core.can_create_carrier() - || (self.awaiting_application_submission && self.pending_local.is_empty()) - || self.fatal - { - return; - } - self.try_create_autonomous_carrier(); - } - - fn try_create_autonomous_carrier_now(&mut self) -> bool { - if !self.mode.is_autonomous() || !self.clock_active { - self.emit_clock_state(); - return false; - } - let allow_no_vote = self.consensus_fallback_allowed_at(Instant::now()); - if !self.core.can_create_carrier() { - self.emit_clock_state(); - return false; - } let creation_time_ns = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() @@ -3282,33 +1428,10 @@ impl ShadowServiceStateV1 { }; match result { Ok((envelope, effects)) => { - self.record_carrier_created(Instant::now()); - let initial_application_payload = application - .as_ref() - .and_then(|application| application.application_payload.clone()); - let assigned_application = application - .as_ref() - .filter(|application| application.acknowledge_assignment) - .map(|application| application.application_header.reference()); - if let Some(application) = &application { - if let Some(metrics) = &self.metrics { - let latency_ns = - creation_time_ns.saturating_sub(application.creation_time_ns); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT, - latency_ns, - 1, - latency_ns, - ); - } + if let Some(application) = application { self.assigned_applications .insert(application.application_header.reference()); } - // The application left the pending queue atomically with the - // successful carrier transition. Publish that state before - // releasing the producer gate or fan-out so observers cannot - // retain a stale nonzero current-depth gauge. - self.record_pipeline_state(); self.emit(ShadowServiceEventV1::Input { kind: if application_round.is_some() { "application_carrier" @@ -3318,25 +1441,8 @@ impl ShadowServiceStateV1 { outcome: "accepted", }); self.report_wal_delta(before); - if let Some(reference) = assigned_application { - self.awaiting_application_submission = true; - self.emit(ShadowServiceEventV1::ApplicationAssigned(reference)); - } - if let Some(application) = application { - if let Err(error) = self.authorize_application( - envelope.reference(), - application.application_header, - application.application_payload, - true, - None, - ShadowApplicationAuthorizationBasisV1::LocallyFixed, - ) { - self.reject(None, error); - } - } - self.broadcast_with_application_payload(&envelope, initial_application_payload); + self.broadcast(&envelope); self.process_effects(effects); - true } Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { if let Some(application) = application { @@ -3350,14 +1456,12 @@ impl ShadowServiceStateV1 { outcome: "waiting_for_quorum", }); self.emit_clock_state(); - false } Err(error) => { if let Some(application) = application { self.pending_local.insert(application.round, application); } self.mark_fatal(error); - false } } } @@ -3368,39 +1472,9 @@ impl ShadowServiceStateV1 { /// so a node behind a continuously advancing committee could never close /// the gap. Healthy rounds still remain paced exclusively by the timer. fn drive_autonomous_catch_up(&mut self) { - if !self.clock_active { - return; - } - for _ in 0..carrier_sync_pipeline_depth(self.committee_size) { - if !self.sync_catch_up || !self.core.can_create_carrier() || self.fatal { - break; - } - let round_before = self.core.local_carrier_round(); - let _ = self.try_create_autonomous_carrier_now(); - if self.core.local_carrier_round() == round_before { - break; - } - } - } - - /// Drain a validity-backed in-window tail without opening an exact repair - /// episode. Every iteration independently requires the exact predecessor - /// quorum and advances at most one physical round. Bound one actor turn so - /// its carrier fan-out remains within the same 64-message credit as exact - /// synchronization (`depth * (n - 1) <= 64`). - fn drive_retained_future_catch_up(&mut self) { - if !self.clock_active || self.fatal { - return; - } - let Some(target) = self.retained_future_target() else { - return; - }; - for _ in 0..carrier_sync_pipeline_depth(self.committee_size) { - if self.core.local_carrier_round() > target || !self.core.local_parent_quorum_ready() { - break; - } + while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { let round_before = self.core.local_carrier_round(); - let _ = self.try_create_autonomous_carrier_now(); + self.try_create_autonomous_carrier(true); if self.core.local_carrier_round() == round_before { break; } @@ -3428,12 +1502,6 @@ impl ShadowServiceStateV1 { ); return; } - if let Some(payload) = &local.application_payload { - if let Err(error) = validate_application_payload_size(payload) { - self.reject(None, error); - return; - } - } if self.mode.is_autonomous() { let application_reference = local.application_header.reference(); if self.assigned_applications.contains(&application_reference) { @@ -3443,18 +1511,8 @@ impl ShadowServiceStateV1 { }); return; } - if let Some(existing) = self.pending_local.get_mut(&local.round) { - if existing.same_application(&local) { - let result = merge_application_payload( - &mut existing.application_payload, - local.application_payload, - application_reference, - ); - existing.acknowledge_assignment |= local.acknowledge_assignment; - if let Err(error) = result { - self.reject(None, error); - return; - } + if let Some(existing) = self.pending_local.get(&local.round) { + if existing == &local { self.emit(ShadowServiceEventV1::Input { kind: "application", outcome: "duplicate", @@ -3472,10 +1530,6 @@ impl ShadowServiceStateV1 { kind: "application", outcome: "queued", }); - // Capture queue occupancy before an immediately available carrier - // slot drains it; the current gauge will return to zero while the - // high-water mark preserves short head-of-line bursts. - self.record_pipeline_state(); self.retry_pending_local(); return; } @@ -3487,18 +1541,8 @@ impl ShadowServiceStateV1 { }); return; } - if let Some(existing) = self.pending_local.get_mut(&local.round) { - if existing.same_application(&local) { - let result = merge_application_payload( - &mut existing.application_payload, - local.application_payload, - existing.application_header.reference(), - ); - existing.acknowledge_assignment |= local.acknowledge_assignment; - if let Err(error) = result { - self.reject(None, error); - return; - } + if let Some(existing) = self.pending_local.get(&local.round) { + if existing == &local { self.emit(ShadowServiceEventV1::Input { kind: "local", outcome: "duplicate", @@ -3533,7 +1577,7 @@ impl ShadowServiceStateV1 { && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(); + self.try_create_autonomous_carrier(false); if self.core.local_carrier_round() == round_before { break; } @@ -3586,9 +1630,6 @@ impl ShadowServiceStateV1 { } fn flush_recovery_requests(&mut self) { - if self.mode.is_autonomous() && !self.clock_active { - return; - } let now = Instant::now(); let mut requests = Vec::new(); for (reference, holders) in &self.pending_recovery { @@ -3616,7 +1657,7 @@ impl ShadowServiceStateV1 { } fn flush_carrier_sync_requests(&mut self, force: bool) { - if !self.mode.is_autonomous() || !self.clock_active { + if !self.mode.is_autonomous() { return; } let round = self.core.local_carrier_round(); @@ -3624,26 +1665,13 @@ impl ShadowServiceStateV1 { if round != self.sync_round { self.sync_round = round; self.sync_round_opened_at = now; + self.sync_last_attempt.clear(); } - self.sync_last_attempt.retain(|(attempt_round, author), _| { - *attempt_round >= round + self.sync_last_attempt.retain(|(author, attempt_round), _| { + *attempt_round == round && self.connected.contains(author) - && self - .core - .authenticated_reference(*author, *attempt_round) - .is_none() + && self.core.admitted_reference(*author, round).is_none() }); - // A complete retained predecessor quorum can advance locally without - // network repair. In particular, suppress an already-expired ordinary - // grace timer while the current actor turn is about to drain that - // tail. A missing quorum still falls through to exact current-slot - // repair after the normal grace interval. - if !force - && self.retained_future_target().is_some() - && self.core.local_parent_quorum_ready() - { - return; - } if !force && !self.sync_catch_up && now.saturating_duration_since(self.sync_round_opened_at) @@ -3651,43 +1679,26 @@ impl ShadowServiceStateV1 { { return; } - let target = self - .sync_catch_up - .then_some(self.sync_catch_up_target) - .flatten(); - let candidates = - carrier_sync_pipeline_slots(round, target, self.committee_size, &self.connected); - for (request_round, author) in candidates { - if self - .core - .authenticated_reference(author, request_round) - .is_some() - { - continue; - } - let slot = (request_round, author); - let should_send = match self.sync_last_attempt.get(&slot) { - Some(last) => { - now.saturating_duration_since(*last) >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 - } - None => self.sync_last_attempt.len() < SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, - }; - if !should_send { - continue; - } - self.sync_last_attempt.insert(slot, now); - #[cfg(test)] - { - self.sync_max_outstanding = - self.sync_max_outstanding.max(self.sync_last_attempt.len()); - } + let requests = self + .connected + .iter() + .copied() + .filter(|author| self.core.admitted_reference(*author, round).is_none()) + .filter(|author| { + self.sync_last_attempt + .get(&(*author, round)) + .is_none_or(|last| { + now.saturating_duration_since(*last) + >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 + }) + }) + .collect::>(); + for author in requests { + self.sync_last_attempt.insert((author, round), now); self.emit(ShadowServiceEventV1::Network { recipient: author, message: NetworkMessage::RbcDagShadowCarrierSyncRequest( - RbcDagShadowCarrierSyncRequest { - author, - round: request_round, - }, + RbcDagShadowCarrierSyncRequest { author, round }, ), }); self.emit(ShadowServiceEventV1::Input { @@ -3695,31 +1706,6 @@ impl ShadowServiceStateV1 { outcome: "sent", }); } - self.finish_catch_up_if_drained(); - } - - fn finish_catch_up_if_drained(&mut self) { - if !self.sync_catch_up { - return; - } - let Some(target) = self.sync_catch_up_target else { - return; - }; - if self.core.local_carrier_round() < target - || !self.sync_last_attempt.is_empty() - || !self.desired_carrier_sync_responses.lock().is_empty() - { - return; - } - self.sync_catch_up = false; - self.sync_catch_up_limit_future = false; - self.sync_catch_up_target = None; - self.catch_up_hint_high_water.clear(); - self.far_future_hint_high_water.clear(); - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_catch_up", - outcome: "target_reached", - }); } fn handle_carrier_sync_request( @@ -3727,13 +1713,6 @@ impl ShadowServiceStateV1 { peer: AuthorityIndex, request: RbcDagShadowCarrierSyncRequest, ) { - if !self.clock_active { - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "paused", - }); - return; - } if request.author != self.own_authority { self.reject( Some(peer), @@ -3752,21 +1731,16 @@ impl ShadowServiceStateV1 { return; } let now = Instant::now(); - let permitted = match self.sync_last_served.get_mut(&peer) { - Some(window) => window.permits(request.round, now), - None => { - self.sync_last_served - .insert(peer, CarrierSyncServeWindowV1::first(request.round, now)); - true - } - }; - if !permitted { + if self.sync_last_served.get(&peer).is_some_and(|(_, last)| { + now.saturating_duration_since(*last) < SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 + }) { self.emit(ShadowServiceEventV1::Input { kind: "carrier_sync_request", outcome: "rate_limited", }); return; } + self.sync_last_served.insert(peer, (request.round, now)); let Some(envelope) = self.core.local_outbound_envelope(request.round) else { self.emit(ShadowServiceEventV1::Input { kind: "carrier_sync_request", @@ -3796,7 +1770,7 @@ impl ShadowServiceStateV1 { peer: AuthorityIndex, response: RbcDagShadowCarrierSyncResponse, ) { - let expected = (response.round, response.author); + let expected = (response.author, response.round); if peer != response.author { self.reject( Some(peer), @@ -3830,7 +1804,7 @@ impl ShadowServiceStateV1 { if response.round < self.core.local_carrier_round() || self .core - .authenticated_reference(response.author, response.round) + .admitted_reference(response.author, response.round) .is_some() { self.sync_last_attempt.remove(&expected); @@ -3851,9 +1825,6 @@ impl ShadowServiceStateV1 { return; } let before = self.core.wal_counts(); - // Exact requested responses always use the normal 64-round - // authenticated window, including while unsolicited proactive - // traffic is narrowed in far catch-up mode. match self.core.receive_or_retain_from_peer( &response.canonical_carrier, &response.authentication_sidecar, @@ -3864,41 +1835,25 @@ impl ShadowServiceStateV1 { ShadowIngressDispositionV1::Authenticated => "authenticated", ShadowIngressDispositionV1::CandidateRetained => "retained_unauthenticated", ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => "ignored", - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => "future_ignored", }; self.emit(ShadowServiceEventV1::Input { kind: "carrier_sync_response", outcome: outcome_label, }); - if let Err(error) = self.observe_carrier_application( - peer, - &response.canonical_carrier, - None, - outcome.disposition(), - ) { - self.reject(Some(peer), error); - } if outcome.disposition() == ShadowIngressDispositionV1::Authenticated && self .core - .authenticated_reference(response.author, response.round) + .admitted_reference(response.author, response.round) == Some(actual_reference) { - // Every fresh process repairs its carrier fixed before - // topology registration at round one. That alone is not - // lag evidence and must not turn healthy startup into a - // permanent exact-sync loop. Later authenticated repairs - // do prove that this clock missed live traffic. - if response.round > 1 { - self.observe_catch_up_hint(peer, response.round, false); - } + self.sync_used_in_open_round = true; } self.report_wal_delta(before); self.process_effects(outcome.effects().to_vec()); self.retry_pending_local(); if self .core - .authenticated_reference(response.author, response.round) + .admitted_reference(response.author, response.round) .is_some() { self.sync_last_attempt.remove(&expected); @@ -3927,45 +1882,6 @@ impl ShadowServiceStateV1 { } } - /// Drain exact-slot repair ahead of the ordinary proactive-carrier FIFO. - /// A globally bounded set of exact `(round, author)` responses is retained - /// by the handle. This is the priority seam a lagging honest node needs to - /// consume pipelined repair while a live quorum continues producing future - /// carriers. - fn reconcile_carrier_sync_responses(&mut self) { - if !self.clock_active { - return; - } - loop { - #[cfg(test)] - { - self.sync_max_desired_responses = self - .sync_max_desired_responses - .max(self.desired_carrier_sync_responses.lock().len()); - } - let current_round = self.core.local_carrier_round(); - let next = { - let mut desired = self.desired_carrier_sync_responses.lock(); - let exact_slot = desired.iter().find_map(|(slot, (_, response))| { - (response.round == current_round).then_some(*slot) - }); - let slot = exact_slot.or_else(|| desired.keys().next().copied()); - slot.and_then(|slot| desired.remove(&slot)) - }; - let Some((peer, response)) = next else { - break; - }; - if let Err(error) = self.validate_peer(peer) { - self.reject(Some(peer), error); - continue; - } - self.handle_carrier_sync_response(peer, response); - if self.fatal { - break; - } - } - } - fn report_new_shadow_deliveries(&mut self, references: &[BlockReference]) { for reference in references { let identity = match self.core.delivery_identity(*reference) { @@ -3987,26 +1903,10 @@ impl ShadowServiceStateV1 { self.emit_comparison_backlog(); match self.core.delivered_application_header(*reference) { Ok(Some((carrier, header))) => { - if let Err(error) = - self.authorize_delivered_application(carrier, header.clone()) - { - self.reject(None, error); - return; - } if self .reported_application_deliveries .insert(header.reference()) { - if let Some(metrics) = &self.metrics { - let latency_ns = current_timestamp_ns() - .saturating_sub(header.meta_creation_time_ns()); - metrics.observe_starfish_rbc_dag_pipeline_latency_ns( - RBC_DAG_LATENCY_CREATION_TO_DELIVERY, - latency_ns, - 1, - latency_ns, - ); - } self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header, @@ -4078,7 +1978,6 @@ fn run_shadow_service( mut state: ShadowServiceStateV1, mut messages: mpsc::Receiver, open_report: ShadowOpenReportV1, - clock_starts_active: bool, ) { if open_report.replayed_batches() != 0 || open_report.discarded_tail_bytes() != 0 { state.emit(ShadowServiceEventV1::Recovered { @@ -4087,42 +1986,20 @@ fn run_shadow_service( }); } state.reconcile_topology(); - state.reconcile_local_applications(); - state.reconcile_verified_application_payloads(); state.reconcile_direct_deliveries(); if !state.observe_external_invalidation() { - if state.mode.is_autonomous() { - // Replay-derived authority and effects must cross the ordered - // event bridge before readiness can release an authoritative - // Core. Clock activation follows Ready in the active-by-default - // path and remains an explicit later message when coordinated. - state.emit_autonomous_recovery_and_ready(&open_report); - if clock_starts_active { - state.activate_clock(); - state.retry_pending_local(); - } - } else { - // Direct mirror mode is observational and preserves its existing - // Ready-first event contract for comparison consumers. - state.emit(ShadowServiceEventV1::Ready { - autonomous_clock: false, - }); - state.emit_recovered_authorized_applications(); - state.emit_comparison_backlog(); - state.process_effects(open_report.recovery_effects().to_vec()); - state.retry_pending_local(); - } - state.drive_consensus_fallback(); + state.emit(ShadowServiceEventV1::Ready { + autonomous_clock: state.mode.is_autonomous(), + }); + state.emit_comparison_backlog(); + state.process_effects(open_report.recovery_effects().to_vec()); + state.retry_pending_local(); if state.mode.is_autonomous() { state.emit_clock_state(); } } while !state.fatal { - state.reconcile_carrier_sync_responses(); - if state.fatal { - break; - } let Some(message) = messages.blocking_recv() else { break; }; @@ -4157,17 +2034,8 @@ fn run_shadow_service( _ => {} } match message { - ShadowServiceMessageV1::ActivateClock(reply) => { - state.activate_clock(); - // A paused open may already hold a recovered or live local - // application. Release that event-driven work immediately - // after the ordered ClockActivated observation; control-only - // production still waits for the fresh heartbeat epoch. - state.retry_pending_local(); - let _ = reply.send(()); - } - ShadowServiceMessageV1::LocalApplicationsChanged => { - state.reconcile_local_applications(); + ShadowServiceMessageV1::LocalCarrier(local) => { + state.enqueue_local(local); } ShadowServiceMessageV1::Carrier { peer, envelope } => { if let Err(error) = state.validate_peer(peer) { @@ -4175,21 +2043,12 @@ fn run_shadow_service( continue; } let before = state.core.wal_counts(); - let open_round_before = state.core.local_carrier_round(); - let catchup_limited = state.sync_catch_up_limit_future; - match state.core.receive_or_retain_from_peer_with_future_window( + match state.core.receive_or_retain_from_peer( &envelope.canonical_carrier, &envelope.authentication_sidecar, peer, - if catchup_limited { - 0 - } else { - EXECUTABLE_MODEL_BUFFER_WINDOW_V1 - }, ) { Ok(outcome) => { - let future_ignored = outcome.disposition() - == ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer; let outcome_label = match outcome.disposition() { ShadowIngressDispositionV1::Authenticated => "authenticated", ShadowIngressDispositionV1::CandidateRetained => { @@ -4198,39 +2057,14 @@ fn run_shadow_service( ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { "ignored" } - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer - if catchup_limited => - { - "catchup_future_ignored" - } - ShadowIngressDispositionV1::IgnoredFutureOutsideBuffer => { - "future_ignored" - } }; state.emit(ShadowServiceEventV1::Input { kind: "carrier", outcome: outcome_label, }); - state.observe_ingress_catch_up_hint( - peer, - &envelope.canonical_carrier, - outcome.disposition(), - open_round_before, - ); - if let Err(error) = state.observe_carrier_application( - peer, - &envelope.canonical_carrier, - envelope.application_payload, - outcome.disposition(), - ) { - state.reject(Some(peer), error); - } state.report_wal_delta(before); state.process_effects(outcome.effects().to_vec()); state.retry_pending_local(); - if future_ignored { - state.flush_carrier_sync_requests(true); - } if outcome.disposition() == ShadowIngressDispositionV1::CandidateRetained { state.reject( Some(peer), @@ -4315,14 +2149,6 @@ fn run_shadow_service( kind: "recovery", outcome: "accepted", }); - if let Err(error) = state.observe_carrier_application( - peer, - &response.canonical_carrier, - None, - ShadowIngressDispositionV1::CandidateRetained, - ) { - state.reject(Some(peer), error); - } state.report_wal_delta(before); state.process_effects(effects); state.retry_pending_local(); @@ -4347,86 +2173,33 @@ fn run_shadow_service( } state.handle_carrier_sync_request(peer, request); } - ShadowServiceMessageV1::CarrierSyncResponsesChanged => {} - ShadowServiceMessageV1::ApplicationPayloadRequest { peer, application } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - state.handle_application_payload_request(peer, application); - } - ShadowServiceMessageV1::ApplicationPayloadResponse { peer, response } => { + ShadowServiceMessageV1::CarrierSyncResponse { peer, response } => { if let Err(error) = state.validate_peer(peer) { state.reject(Some(peer), error); continue; } - if let Err(error) = state.handle_application_payload_response(peer, response) { - state.reject(Some(peer), error); - } + state.handle_carrier_sync_response(peer, response); } - ShadowServiceMessageV1::VerifiedApplicationPayloadsChanged => {} ShadowServiceMessageV1::DirectDeliveriesChanged => { state.reconcile_direct_deliveries(); } - ShadowServiceMessageV1::TopologyChanged => {} + ShadowServiceMessageV1::TopologyChanged => state.reconcile_topology(), ShadowServiceMessageV1::RetryRecovery => { - state.awaiting_application_submission = false; - state.reconcile_local_applications(); + state.reconcile_topology(); state.reconcile_pending_recovery(); state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); - state.flush_application_payload_requests(); - } - ShadowServiceMessageV1::HeartbeatTick => { - state.awaiting_application_submission = false; - state.try_create_autonomous_carrier(); - } - ShadowServiceMessageV1::NormalCarrierDeadline { generation } => { - state.observe_normal_carrier_deadline(generation); - } - ShadowServiceMessageV1::ConsensusTimeoutDeadline { generation, slot } => { - state.observe_consensus_timeout_deadline(generation, slot); } + ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(true), ShadowServiceMessageV1::DataAvailabilityChanged => { state.reconcile_data_availability(); } - #[cfg(test)] - ShadowServiceMessageV1::InspectRbcProgress(reply) => { - let progress = ( - state.core.optimistic_promise_count(), - state - .core - .certified_delivery_count() - .expect("test progress inspection requires unambiguous deliveries"), - ); - let _ = reply.send(progress); - } - #[cfg(test)] - ShadowServiceMessageV1::InspectCarrierSync(reply) => { - let desired_responses = state.desired_carrier_sync_responses.lock().len(); - state.sync_max_desired_responses = - state.sync_max_desired_responses.max(desired_responses); - let _ = reply.send(CarrierSyncInspectionV1 { - open_round: state.core.local_carrier_round(), - target: state.sync_catch_up_target, - outstanding: state.sync_last_attempt.len(), - desired_responses, - max_outstanding: state.sync_max_outstanding, - max_desired_responses: state.sync_max_desired_responses, - }); - } ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); - state.reconcile_local_applications(); - state.reconcile_carrier_sync_responses(); - state.reconcile_verified_application_payloads(); state.reconcile_direct_deliveries(); - state.drive_consensus_fallback(); state.drive_autonomous_catch_up(); - state.drive_retained_future_catch_up(); state.flush_carrier_sync_requests(false); - state.flush_application_payload_requests(); } let events = state.events.clone(); if let Err(error) = state.core.shutdown() { @@ -4444,29 +2217,6 @@ fn delivery_slot(identity: &ShadowDeliveryIdentityV1) -> ShadowDeliverySlotV1 { } } -fn current_timestamp_ns() -> TimestampNs { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - .try_into() - .unwrap_or(TimestampNs::MAX) -} - -fn latency_since_header_creation<'a>( - headers: impl Iterator, - now_ns: TimestampNs, -) -> (u64, u64, u64) { - headers.fold((0u64, 0u64, 0u64), |(total, samples, maximum), header| { - let latency = now_ns.saturating_sub(header.meta_creation_time_ns()); - ( - total.saturating_add(latency), - samples.saturating_add(1), - maximum.max(latency), - ) - }) -} - fn authentication_sidecar_size(scheme: BlockAuthenticationScheme, committee_size: usize) -> usize { const SIDECAR_HEADER_SIZE: usize = 3; SIDECAR_HEADER_SIZE @@ -4513,55 +2263,6 @@ fn validate_wire_size( } } -fn validate_application_payload_size( - payload: &TransactionData, -) -> Result<(), ShadowServiceErrorV1> { - let actual = bincode::serialized_size(payload) - .map_err(|error| ShadowServiceErrorV1::ApplicationPayloadSerialization(error.to_string()))? - .try_into() - .unwrap_or(usize::MAX); - validate_wire_size( - "application payload", - actual, - SHADOW_APPLICATION_PAYLOAD_MAX_SIZE_V1, - ) -} - -fn application_payloads_equal( - left: Option<&TransactionData>, - right: Option<&TransactionData>, -) -> bool { - match (left, right) { - (Some(left), Some(right)) => left.transactions() == right.transactions(), - (None, None) => true, - (Some(_), None) | (None, Some(_)) => false, - } -} - -fn merge_application_payload( - retained: &mut Option>, - incoming: Option>, - application: BlockReference, -) -> Result<(), ShadowServiceErrorV1> { - let Some(incoming) = incoming else { - return Ok(()); - }; - match retained { - Some(existing) - if !application_payloads_equal(Some(existing.as_ref()), Some(incoming.as_ref())) => - { - Err(ShadowServiceErrorV1::ConflictingApplicationPayload( - application, - )) - } - Some(_) => Ok(()), - None => { - *retained = Some(incoming); - Ok(()) - } - } -} - fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { matches!( error, @@ -4571,16 +2272,8 @@ fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { #[cfg(test)] mod tests { - use std::{ - collections::VecDeque, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering as AtomicOrdering}, - }, - time::Duration, - }; + use std::{sync::Arc, time::Duration}; - use prometheus::Registry; use tempfile::TempDir; use tokio::time::timeout; @@ -4588,877 +2281,47 @@ mod tests { use crate::{ committee::Committee, crypto::{TransactionsCommitment, mac_keyrings_for_test}, - encoder::{Encoder, ShardEncoder}, starfish_rbc_dag::{ - CandidateCarrierV1, CarrierAuthorizerV1, CarrierHeaderV1Args, LeaderChoiceV1, - RbcDagProtocolInstanceId, RbcPhaseStatementV1, carrier_genesis_reference, + CandidateCarrierV1, CarrierAuthorizerV1, CarrierHeaderV1Args, RbcDagProtocolInstanceId, + RbcPhaseStatementV1, carrier_genesis_reference, }, - types::{BaseTransaction, BlockDigest, Transaction, VerifiedBlock}, + types::{BlockDigest, VerifiedBlock}, }; const N: usize = 4; const EVENT_TIMEOUT: Duration = Duration::from_secs(5); - #[test] - fn carrier_sync_limiter_is_order_independent_and_throttles_replay() { - let start = Instant::now(); - let first = SHADOW_CARRIER_SYNC_MAX_ADVANCING_BURST_V1 as RoundNumber; - let mut limiter = CarrierSyncServeWindowV1::first(first, start); - for round in (1..first).rev() { - assert!(limiter.permits(round, start)); - } - let next = first.saturating_add(1); - assert!(!limiter.permits(next, start)); - assert!(!limiter.permits(1, start)); - - let later = start + SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1; - assert!(limiter.permits(next, later)); - assert!(!limiter.permits(next, later)); - assert!(limiter.permits(1, later)); - } - - #[test] - fn ten_validator_sync_window_is_round_major_and_bounded_to_sixty_three_slots() { - let connected = (1..10) - .map(|authority| authority as AuthorityIndex) - .collect(); - let slots = carrier_sync_pipeline_slots(40, Some(100), 10, &connected); - - assert_eq!(carrier_sync_pipeline_depth(10), 7); - assert_eq!(slots.len(), 63); - assert_eq!(slots.first(), Some(&(40, 1))); - assert_eq!(slots.last(), Some(&(46, 9))); - assert!(slots.windows(2).all(|pair| pair[0] < pair[1])); - assert!(!slots.iter().any(|(round, _)| *round >= 47)); - assert!(slots.len() <= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1); - - let moderate = carrier_sync_pipeline_slots(40, None, 10, &connected); - assert_eq!(moderate.len(), 9); - assert!(moderate.iter().all(|(round, _)| *round == 40)); + struct Harness { + _directory: TempDir, + committee: RbcDagCommitteeContextV1, + context: RbcDagContextV1, + keyrings: Vec>, + paths: Vec, } - #[test] - fn exact_sync_responses_coalesce_by_slot_without_overwrite() { - let (sender, _receiver) = mpsc::channel(1); - let desired = Arc::new(Mutex::new(BTreeMap::new())); - let handle = StarfishRbcDagShadowServiceHandleV1 { - sender, - max_sidecar_size: 3 + N * MAC_TAG_SIZE, - own_authority: 0, - committee_size: N, - input_capacity: 1, - mode: ShadowServiceModeV1::AutonomousClock { - heartbeat_interval: Duration::from_secs(1), - }, - desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), - desired_carrier_sync_responses: Arc::clone(&desired), - desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), - desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), - invalidated_by_overload: Arc::new(Mutex::new(None)), - }; - let response = |author, round, marker| RbcDagShadowCarrierSyncResponse { - author, - round, - canonical_carrier: vec![marker], - authentication_sidecar: Vec::new(), - }; - - let first = response(1, 7, 0xA1); - let later = response(1, 8, 0xA2); - handle.carrier_sync_response(1, first.clone()).unwrap(); - handle.carrier_sync_response(1, later.clone()).unwrap(); - handle.carrier_sync_response(1, first.clone()).unwrap(); - assert_eq!(desired.lock().len(), 2); - assert_eq!(desired.lock().get(&(7, 1)).unwrap().1, first); - assert_eq!(desired.lock().get(&(8, 1)).unwrap().1, later); - - assert!(matches!( - handle.carrier_sync_response(1, response(1, 7, 0xFF)), - Err(ShadowServiceErrorV1::UnexpectedSyncResponse { - author: 1, - round: 7, - }) - )); - assert_eq!( - desired.lock().get(&(7, 1)).unwrap().1.canonical_carrier, - vec![0xA1] - ); - - 'fill: for round in 1..=RoundNumber::MAX { - for author in 1..N as AuthorityIndex { - if desired.lock().len() == SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 { - break 'fill; - } - if desired.lock().contains_key(&(round, author)) { - continue; - } - handle - .carrier_sync_response(author, response(author, round, author as u8)) - .unwrap(); - } + impl Harness { + fn new() -> Self { + Self::new_with_n(N) } - assert_eq!(desired.lock().len(), SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1); - assert!(matches!( - handle.carrier_sync_response(1, response(1, 10_000, 0xCC)), - Err(ShadowServiceErrorV1::CarrierSyncResponseCapacity { - capacity: SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, - }) - )); - } - - #[tokio::test] - async fn catch_up_target_is_the_monotone_validity_stake_high_water() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (state, _events, _message_rx, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); - tokio::task::spawn_blocking(move || { - let mut state = state; - state.observe_catch_up_hint(1, 100, true); - assert_eq!(state.sync_catch_up_target, None); - state.observe_catch_up_hint(2, 70, true); - assert_eq!(state.sync_catch_up_target, Some(70)); - assert!(state.sync_catch_up); - assert!(state.sync_catch_up_limit_future); - - state.observe_catch_up_hint(3, 90, true); - assert_eq!(state.sync_catch_up_target, Some(90)); - state.observe_catch_up_hint(2, 110, true); - assert_eq!(state.sync_catch_up_target, Some(100)); - state.observe_catch_up_hint(1, 80, true); - assert_eq!(state.sync_catch_up_target, Some(100)); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - } - #[tokio::test] - async fn in_window_future_hints_do_not_start_exact_catch_up() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (state, _events, _message_rx, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); - tokio::task::spawn_blocking(move || { - let mut state = state; - state.observe_catch_up_hint(1, 40, false); - assert_eq!(state.retained_future_target(), None); - // Two remote hints carry validity stake in the four-node test, - // but both remain inside normal authenticated retention. - state.observe_catch_up_hint(2, 40, false); - assert_eq!( - state.validity_backed_high_water(&state.catch_up_hint_high_water), - Some(40) + fn new_with_n(n: usize) -> Self { + let committee = Committee::new_test(vec![1; n]); + let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); + let context = RbcDagContextV1::new_with_committee( + RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), + &committee, + BlockAuthenticationScheme::MacVector, ); - assert_eq!(state.retained_future_target(), Some(40)); - assert!(!state.sync_catch_up); - assert!(!state.sync_catch_up_limit_future); - assert_eq!(state.sync_catch_up_target, None); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - } - - fn producer_carrier( - committee: &RbcDagCommitteeContextV1, - chains: &[Vec], - author: AuthorityIndex, - round: RoundNumber, - ) -> CandidateCarrierV1 { - let previous = |authority: AuthorityIndex| { - if round == 1 { - carrier_genesis_reference(authority) - } else { - chains[authority as usize][round as usize - 2] - } - }; - let mut parent_stake = committee - .committee() - .get_stake(author) - .expect("producer authority belongs to the committee"); - let mut weak_parents = Vec::new(); - for parent_authority in committee.committee().authorities() { - // Authority zero is the lagger. A live quorum of the other - // producers must remain able to extend without naming its stale - // physical chain. - if parent_authority == 0 || parent_authority == author { - continue; - } - weak_parents.push(previous(parent_authority)); - parent_stake = parent_stake.saturating_add( - committee - .committee() - .get_stake(parent_authority) - .expect("parent authority belongs to the committee"), - ); - if parent_stake >= committee.committee().quorum_threshold() { - break; - } - } - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: round, - own_prev: previous(author), - weak_parents, - transactions_commitment: TransactionsCommitment::default(), - application_header: None, - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: round.into(), - }, - committee, - ) - .unwrap() - } - - fn feed_producer_round( - state: &mut ShadowServiceStateV1, - harness: &Harness, - chains: &mut [Vec], - round: RoundNumber, - ) { - let open_before = state.core.local_carrier_round(); - for author in 1..state.committee_size as AuthorityIndex { - let candidate = producer_carrier(&harness.committee, chains, author, round); - let reference = candidate.reference(); - let envelope = harness.envelope(&candidate, author); - let outcome = state - .core - .receive_or_retain_from_peer( - &envelope.canonical_carrier, - &envelope.authentication_sidecar, - author, - ) - .unwrap(); - assert_eq!( - outcome.disposition(), - ShadowIngressDispositionV1::Authenticated - ); - state.observe_ingress_catch_up_hint( - author, - &envelope.canonical_carrier, - outcome.disposition(), - open_before, - ); - state.process_effects(outcome.effects().to_vec()); - chains[author as usize].push(reference); - } - } - - async fn assert_retained_future_tail_closes_without_exact_sync(n: usize, initial_tail: u32) { - let harness = Harness::new_with_n(n); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (mut state, mut events, _messages, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - let sync_requests = Arc::new(AtomicUsize::new(0)); - let sync_requests_observed = Arc::clone(&sync_requests); - let event_drain = tokio::spawn(async move { - while let Some(event) = events.recv().await { - if matches!( - event, - ShadowServiceEventV1::Network { - message: NetworkMessage::RbcDagShadowCarrierSyncRequest(_), - .. - } - ) { - sync_requests_observed.fetch_add(1, AtomicOrdering::Relaxed); - } - } - }); - - tokio::task::spawn_blocking(move || { - let mut chains = vec![Vec::new(); n]; - state.connected = (1..n as AuthorityIndex).collect(); - state.clock_active = false; - for round in 1..=initial_tail { - feed_producer_round(&mut state, &harness, &mut chains, round); - } - assert_eq!(state.core.local_carrier_round(), 1); - assert_eq!(state.retained_future_target(), Some(initial_tail)); - - state.activate_clock(); - state.sync_round_opened_at = Instant::now() - Duration::from_secs(60 * 60 * 3); - let depth = carrier_sync_pipeline_depth(n) as RoundNumber; - let mut healthy_round = initial_tail; - for _ in 0..initial_tail.saturating_add(4) { - // Keep the healthy quorum producing while the lagger consumes - // its retained tail. - healthy_round = healthy_round.saturating_add(1); - feed_producer_round(&mut state, &harness, &mut chains, healthy_round); - let before = state.core.local_carrier_round(); - state.drive_retained_future_catch_up(); - let after = state.core.local_carrier_round(); - assert!(after.saturating_sub(before) <= depth); - state.flush_carrier_sync_requests(false); - if healthy_round.saturating_add(1).saturating_sub(after) <= 2 { - break; - } - } - let gap = healthy_round - .saturating_add(1) - .saturating_sub(state.core.local_carrier_round()); - assert!(gap <= 2, "retained local catch-up plateaued with gap {gap}"); - assert!(!state.sync_catch_up); - assert!(!state.sync_catch_up_limit_future); - - let buffered_capacity = EXECUTABLE_MODEL_BUFFER_WINDOW_V1 - .saturating_sub(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) - as usize - * n.saturating_sub(1); - assert!(state.core.buffered_authenticated_carrier_count() <= buffered_capacity); - - // Stop producers and prove the target-carrier off-by-one: fixing - // the hinted carrier round opens its successor. - if let Some(final_target) = state.retained_future_target() { - for _ in 0..initial_tail.saturating_add(4) { - state.drive_retained_future_catch_up(); - if state.core.local_carrier_round() > final_target { - break; - } - } - assert_eq!(state.core.local_carrier_round(), final_target + 1); - } - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - event_drain.await.unwrap(); - assert_eq!(sync_requests.load(AtomicOrdering::Relaxed), 0); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn four_node_retained_gap_seventeen_closes_without_exact_sync() { - assert_retained_future_tail_closes_without_exact_sync(4, 17).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn ten_node_retained_gap_thirty_two_closes_without_exact_sync() { - assert_retained_future_tail_closes_without_exact_sync(10, 32).await; - } - - #[tokio::test] - async fn requested_future_response_uses_normal_window_during_far_catch_up() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (state, _events, _message_rx, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); - let round = 10; - let previous = |authority: AuthorityIndex| BlockReference { - authority, - round: round - 1, - digest: BlockDigest::from([0x70 + authority as u8; 32]), - }; - let candidate = CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author: 1, - carrier_round: round, - own_prev: previous(1), - weak_parents: [0, 2].into_iter().map(previous).collect(), - transactions_commitment: TransactionsCommitment::default(), - application_header: None, - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: round.into(), - }, - &harness.committee, - ) - .unwrap(); - let reference = candidate.reference(); - let envelope = harness.envelope(&candidate, 1); - tokio::task::spawn_blocking(move || { - let mut state = state; - state.sync_catch_up_limit_future = true; - state.sync_last_attempt.insert((round, 1), Instant::now()); - state.handle_carrier_sync_response( - 1, - RbcDagShadowCarrierSyncResponse { - author: 1, - round, - canonical_carrier: envelope.canonical_carrier, - authentication_sidecar: envelope.authentication_sidecar, - }, - ); - - assert_eq!(state.core.local_carrier_round(), 1); - assert_eq!( - state.core.authenticated_reference(1, round), - Some(reference) - ); - assert!(!state.sync_last_attempt.contains_key(&(round, 1))); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - } - - #[test] - fn consensus_pacemaker_fixed_grid_does_not_inherit_c2_for_a_new_slot() { - let timeout = Duration::from_millis(100); - let origin = Instant::now(); - let mut pacemaker = ConsensusPacemakerV1::new(2); - - assert!(!pacemaker.fallback_allowed(2, true, false, timeout, origin)); - assert!(pacemaker.fallback_allowed(2, true, false, timeout, origin + timeout)); - assert!( - !pacemaker.fallback_allowed(3, true, false, timeout, origin + timeout), - "a fixed-grid tick at the instant slot 3 opens must reset C2" - ); - assert!(!pacemaker.fallback_allowed( - 3, - true, - false, - timeout, - origin + timeout + timeout - Duration::from_nanos(1), - )); - assert!(pacemaker.fallback_allowed(3, true, false, timeout, origin + timeout + timeout,)); - } - - #[test] - fn consensus_pacemaker_c2_starts_only_when_a1_becomes_ready() { - let timeout = Duration::from_millis(100); - let origin = Instant::now(); - let mut pacemaker = ConsensusPacemakerV1::new(4); - - assert!(!pacemaker.fallback_allowed(4, false, false, timeout, origin)); - assert!(!pacemaker.fallback_allowed( - 4, - false, - false, - timeout, - origin + timeout.saturating_mul(10), - )); - let a1_at = origin + timeout.saturating_mul(10); - assert!(!pacemaker.fallback_allowed(4, true, false, timeout, a1_at)); - assert!(!pacemaker.fallback_allowed( - 4, - true, - false, - timeout, - a1_at + timeout - Duration::from_nanos(1), - )); - assert!(pacemaker.fallback_allowed(4, true, false, timeout, a1_at + timeout)); - } - - #[test] - fn consensus_pacemaker_c3_authorizes_immediate_slot_bound_catch_up() { - let timeout = Duration::from_secs(60); - let origin = Instant::now(); - let mut pacemaker = ConsensusPacemakerV1::new(7); - - assert!(pacemaker.fallback_allowed(7, false, true, timeout, origin)); - assert!( - !pacemaker.fallback_allowed(8, false, false, timeout, origin), - "C3 evidence for slot 7 must not leak into slot 8" - ); - } - - #[tokio::test] - async fn service_c3_emits_a_control_carrier_without_waiting_for_the_fixed_grid() { - let harness = Harness::new(); - let (mut core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let reference = |authority, consensus_round, marker| { - ConsensusVertexReference::new( - BlockReference { - authority, - round: 100 + consensus_round, - digest: BlockDigest::from([marker; 32]), - }, - consensus_round, - ) - }; - - let round_one_leader = reference(1, 1, 0x11); - core.inject_projected_consensus_for_test( - round_one_leader, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }, - ); - for (author, choice) in [ - ( - 0, - LeaderChoiceV1::Vote { - leader: round_one_leader, - }, - ), - ( - 1, - LeaderChoiceV1::Vote { - leader: round_one_leader, - }, - ), - ( - 3, - LeaderChoiceV1::NoVote { - leader_author: 1, - leader_round: 1, - }, - ), - ] { - core.inject_projected_consensus_for_test( - reference(author, 2, 0x20 + author as u8), - vec![round_one_leader], - choice, - ); - } - for author in [1, 2, 3] { - core.inject_projected_consensus_for_test( - reference(author, 3, 0x30 + author as u8), - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: 2, - leader_round: 2, - }, - ); - } - core.set_next_local_consensus_round_for_test(3); - assert!(core.has_projected_consensus_quorum(3)); - - let (state, mut event_rx, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - - let state = tokio::task::spawn_blocking(move || { - let mut state = state; - state.drive_consensus_fallback(); - state - }) - .await - .unwrap(); - assert_eq!(state.core.next_local_consensus_round(), 4); - let mut emitted = None; - while let Ok(event) = event_rx.try_recv() { - if let ShadowServiceEventV1::Network { - message: NetworkMessage::RbcDagShadowCarrier(carrier), - .. - } = event - { - emitted = Some(carrier); - break; - } - } - let emitted = emitted.expect("C3 must schedule a carrier immediately"); - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &emitted.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - let vertex = candidate - .header() - .consensus_vertex() - .expect("C3 carrier must contain the lagging logical slot"); - assert_eq!(vertex.consensus_round(), 3); - assert_eq!( - vertex.leader_choice(), - LeaderChoiceV1::NoVote { - leader_author: 2, - leader_round: 2, - } - ); - state.core.shutdown().unwrap(); - } - - #[tokio::test] - async fn service_c2_deadline_message_creates_without_waiting_for_the_next_grid_tick() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let leader_timeout = Duration::from_millis(20); - let (mut state, mut event_rx, mut message_rx, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), leader_timeout); - - state.refresh_consensus_pacemaker(); - assert_eq!(state.core.local_carrier_round(), 1); - let (generation, slot) = match timeout(Duration::from_secs(1), message_rx.recv()) - .await - .expect("slot-bound C2 deadline was not scheduled") - { - Some(ShadowServiceMessageV1::ConsensusTimeoutDeadline { generation, slot }) => { - (generation, slot) - } - other => panic!( - "unexpected deadline message: {:?}", - other.map(|message| message.kind()) - ), - }; - assert_eq!(slot, 1); - state.observe_consensus_timeout_deadline(generation, slot); - assert!(state.consensus_pacemaker.c2_timed_out); - - let state = tokio::task::spawn_blocking(move || { - state.drive_consensus_fallback(); - state - }) - .await - .unwrap(); - assert!(!state.core.can_create_carrier()); - assert_eq!(state.core.next_local_consensus_round(), 2); - let mut emitted = false; - while let Ok(event) = event_rx.try_recv() { - emitted |= matches!( - event, - ShadowServiceEventV1::Network { - message: NetworkMessage::RbcDagShadowCarrier(_), - .. - } - ); - } - assert!(emitted, "C2 deadline must schedule a carrier immediately"); - state.core.shutdown().unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn consensus_timeout_deadline_coalesces_and_rejects_stale_wakes() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (mut state, _event_rx, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - - state.refresh_consensus_pacemaker(); - let scheduled = state - .consensus_timeout_deadline - .expect("C2 readiness did not arm its persistent deadline"); - assert_eq!(scheduled.slot, 1); - - // Repeated ingress/maintenance observations for one logical slot - // share one generation and one physical timer. - for _ in 0..8 { - state.refresh_consensus_pacemaker(); - assert_eq!(state.consensus_timeout_deadline, Some(scheduled)); - assert_eq!(state.consensus_timeout_generation, scheduled.generation); - } - - state.observe_consensus_timeout_deadline( - scheduled.generation.wrapping_sub(1), - scheduled.slot, - ); - assert_eq!(state.consensus_timeout_deadline, Some(scheduled)); - assert!(!state.consensus_pacemaker.c2_timed_out); - - // Make the unit-state deadline due without sleeping for an hour. The - // current generation is consumed once; a duplicate queued wake is a - // no-op and cannot authorize a different slot. - let due = Instant::now() - .checked_sub(Duration::from_millis(1)) - .unwrap_or_else(Instant::now); - let due_scheduled = ConsensusTimeoutDeadlineV1 { - deadline: due, - ..scheduled - }; - state.consensus_timeout_deadline = Some(due_scheduled); - state - .consensus_timeout_deadline_tx - .send_replace(Some(due_scheduled)); - state.observe_consensus_timeout_deadline(due_scheduled.generation, due_scheduled.slot); - assert_eq!(state.consensus_timeout_deadline, None); - assert!(state.consensus_pacemaker.c2_timed_out); - state.observe_consensus_timeout_deadline(due_scheduled.generation, due_scheduled.slot); - assert_eq!(state.consensus_timeout_deadline, None); - - // Replacing the actor-owned target leaves exactly one desired - // deadline. Advancing the logical slot cancels it, and its already - // queued generation remains harmless. - let replacement_deadline = Instant::now() + Duration::from_secs(60 * 60); - state.schedule_consensus_timeout_deadline(1, replacement_deadline); - let replacement = state.consensus_timeout_deadline.unwrap(); - assert_ne!(replacement.generation, scheduled.generation); - state.core.set_next_local_consensus_round_for_test(2); - state.refresh_consensus_pacemaker(); - assert_eq!(state.consensus_pacemaker.slot, 2); - assert_eq!(state.consensus_timeout_deadline, None); - state.observe_consensus_timeout_deadline(replacement.generation, replacement.slot); - assert_eq!(state.consensus_timeout_deadline, None); - - state.core.shutdown().unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn normal_carrier_deadline_coalesces_and_rejects_stale_generations() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (mut state, mut event_rx, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - - tokio::task::spawn_blocking(move || { - state.normal_carrier_min_spacing = Duration::from_secs(60 * 60); - state.try_create_autonomous_carrier(); - assert_eq!(state.core.local_carrier_round(), 1); - assert!(!state.core.can_create_carrier()); - - // Every normal trigger while the permit is closed must share one - // generation and one deadline, regardless of its source. - state.try_create_autonomous_carrier(); - let scheduled = state - .normal_carrier_deadline - .expect("closed permit did not schedule its persistent wake"); - assert!(state.normal_carrier_requested); - for _ in 0..8 { - state.try_create_autonomous_carrier(); - assert_eq!(state.normal_carrier_deadline, Some(scheduled)); - assert_eq!(state.normal_carrier_generation, scheduled.generation); - } - - let stale_generation = scheduled.generation.wrapping_sub(1); - state.observe_normal_carrier_deadline(stale_generation); - assert_eq!(state.normal_carrier_deadline, Some(scheduled)); - assert!(state.normal_carrier_requested); - - // Admit the exact predecessor quorum so the current wake can fix - // one, and only one, successor carrier. Move the deterministic - // unit-state deadline to the past instead of sleeping for an hour. - let mut chains = vec![Vec::new(); N]; - feed_producer_round(&mut state, &harness, &mut chains, 1); - assert_eq!(state.core.local_carrier_round(), 2); - assert!(state.core.can_create_carrier()); - let due = Instant::now() - .checked_sub(Duration::from_millis(1)) - .unwrap_or_else(Instant::now); - let due_scheduled = NormalCarrierDeadlineV1 { - generation: scheduled.generation, - deadline: due, - }; - state.normal_carrier_next_allowed_at = Some(due); - state.normal_carrier_deadline = Some(due_scheduled); - state - .normal_carrier_deadline_tx - .send_replace(Some(due_scheduled)); - - state.observe_normal_carrier_deadline(due_scheduled.generation); - assert_eq!(state.core.local_carrier_round(), 2); - assert!(!state.core.can_create_carrier()); - assert!(!state.normal_carrier_requested); - assert_eq!(state.normal_carrier_deadline, None); - - // A duplicate queued wake for the generation just consumed is a - // no-op and cannot spend another physical slot. - state.observe_normal_carrier_deadline(due_scheduled.generation); - assert_eq!(state.core.local_carrier_round(), 2); - assert!(!state.core.can_create_carrier()); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - - let proactive_carriers = std::iter::from_fn(|| event_rx.try_recv().ok()) - .filter(|event| { - matches!( - event, - ShadowServiceEventV1::Network { - message: NetworkMessage::RbcDagShadowCarrier(_), - .. - } - ) - }) - .count(); - assert_eq!(proactive_carriers, 2 * (N - 1)); - } - - struct Harness { - _directory: TempDir, - committee: RbcDagCommitteeContextV1, - context: RbcDagContextV1, - keyrings: Vec>, - paths: Vec, - } - - impl Harness { - fn new() -> Self { - Self::new_with_n(N) - } - - fn new_with_n(n: usize) -> Self { - let committee = Committee::new_test(vec![1; n]); - let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ); - let directory = tempfile::tempdir().unwrap(); - let paths = (0..n) - .map(|authority| directory.path().join(format!("shadow-{authority}.wal"))) - .collect(); - Self { - _directory: directory, - committee, - context, - keyrings: mac_keyrings_for_test(n), - paths, + let directory = tempfile::tempdir().unwrap(); + let paths = (0..n) + .map(|authority| directory.path().join(format!("shadow-{authority}.wal"))) + .collect(); + Self { + _directory: directory, + committee, + context, + keyrings: mac_keyrings_for_test(n), + paths, } } @@ -5533,31 +2396,6 @@ mod tests { .unwrap() } - fn start_autonomous_paused_with_interval( - &self, - authority: AuthorityIndex, - heartbeat_interval: Duration, - ) -> ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ) { - start_starfish_rbc_dag_shadow_service_with_mode_v1( - &self.paths[authority as usize], - self.committee.clone(), - authority, - self.context, - ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), - Vec::new(), - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - ShadowWalSyncPolicyV1::EveryBatch, - None, - None, - false, - ) - .unwrap() - } - fn envelope( &self, candidate: &CandidateCarrierV1, @@ -5577,102 +2415,10 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: candidate.canonical_wire_bytes().unwrap(), authentication_sidecar: authentication.canonical_wire_bytes(), - application_payload: None, } } } - fn standalone_autonomous_state( - core: StarfishRbcDagShadowV1, - committee: RbcDagCommitteeContextV1, - leader_timeout: Duration, - ) -> ( - ShadowServiceStateV1, - mpsc::Receiver, - mpsc::Receiver, - mpsc::Sender, - ) { - let slot = core.next_local_consensus_round(); - let committee_size = committee.committee().len(); - let (event_tx, event_rx) = mpsc::channel(128); - let (message_tx, message_rx) = mpsc::channel(8); - let (clock_activation_tx, _clock_activation_rx) = watch::channel(true); - let (normal_carrier_deadline_tx, _normal_carrier_deadline_rx) = watch::channel(None); - let (consensus_timeout_deadline_tx, consensus_timeout_deadline_rx) = watch::channel(None); - spawn_consensus_timeout_deadline_task( - consensus_timeout_deadline_rx, - message_tx.downgrade(), - ); - let state = ShadowServiceStateV1 { - core, - committee, - mode: ShadowServiceModeV1::AutonomousClock { - heartbeat_interval: leader_timeout, - }, - clock_active: true, - clock_activation_tx, - wal_sync_policy: ShadowWalSyncPolicyV1::EveryBatch, - metrics: None, - own_authority: 0, - committee_size, - events: event_tx, - connected: BTreeSet::new(), - catch_up_hint_high_water: BTreeMap::new(), - far_future_hint_high_water: BTreeMap::new(), - desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), - desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), - desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), - desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), - observed_topology: BTreeMap::new(), - invalidated_by_overload: Arc::new(Mutex::new(None)), - pending_local: BTreeMap::new(), - assigned_applications: BTreeSet::new(), - pending_data_availability: BTreeSet::new(), - pending_recovery: BTreeMap::new(), - recovery_last_attempt: BTreeMap::new(), - sync_last_attempt: BTreeMap::new(), - sync_last_served: BTreeMap::new(), - authorized_applications: BTreeMap::new(), - quarantined_application_payloads: BTreeMap::new(), - payload_last_served: BTreeMap::new(), - sync_round: 1, - sync_round_opened_at: Instant::now(), - sync_catch_up: false, - sync_catch_up_limit_future: false, - sync_catch_up_target: None, - sync_max_outstanding: 0, - sync_max_desired_responses: 0, - awaiting_application_submission: false, - consensus_pacemaker: ConsensusPacemakerV1::new(slot), - // Existing state-level tests invoke creation synchronously and do - // not run the service deadline task. Keep their historical - // immediate behavior unless a pacer test overrides this field. - normal_carrier_min_spacing: Duration::ZERO, - normal_carrier_next_allowed_at: None, - normal_carrier_requested: false, - normal_carrier_generation: 0, - normal_carrier_deadline: None, - normal_carrier_deadline_tx, - consensus_timeout_generation: 0, - consensus_timeout_deadline: None, - consensus_timeout_deadline_tx, - retry_notification_pending: Arc::new(AtomicBool::new(false)), - heartbeat_notification_pending: Arc::new(AtomicBool::new(false)), - direct_deliveries: BTreeSet::new(), - reported_shadow_deliveries: BTreeSet::new(), - reported_application_deliveries: BTreeSet::new(), - recovered_shadow_deliveries: BTreeSet::new(), - comparison_backlog: ShadowComparisonBacklogV1::new(BTreeSet::new()), - reported_matches: BTreeSet::new(), - reported_mismatches: BTreeSet::new(), - reported_conflicts: BTreeSet::new(), - fatal: false, - }; - (state, event_rx, message_rx, message_tx) - } - async fn next_event(events: &mut mpsc::Receiver) -> ShadowServiceEventV1 { timeout(EVENT_TIMEOUT, events.recv()) .await @@ -5692,66 +2438,6 @@ mod tests { } } - #[tokio::test] - async fn recovered_control_frontier_precedes_ready_and_clock_activation() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (mut state, mut events, _messages, _message_tx) = - standalone_autonomous_state(core, harness.committee.clone(), Duration::from_secs(60)); - state.clock_active = false; - let delta = CommittedFrontierDeltaV1 { - output_sequence: 1, - anchor: ConsensusVertexReference::new(BlockReference::new_test(1, 10), 7), - frontier: vec![None; harness.committee.committee().len()], - carriers: Vec::new(), - applications: Vec::new(), - application_diagnostics: Vec::new(), - }; - let report = - ShadowOpenReportV1::with_recovered_committed_frontiers_for_test(vec![delta.clone()]); - - state = tokio::task::spawn_blocking(move || { - state.emit_autonomous_recovery_and_ready(&report); - state - }) - .await - .unwrap(); - assert!(matches!( - next_event(&mut events).await, - ShadowServiceEventV1::FrontierCommitted(actual) if actual == delta - )); - loop { - match next_event(&mut events).await { - ShadowServiceEventV1::Ready { - autonomous_clock: true, - } => break, - ShadowServiceEventV1::PendingRecovery(_) - | ShadowServiceEventV1::ClockState { .. } => {} - unexpected => panic!("unexpected recovery-prefix event: {unexpected:?}"), - } - } - assert!(events.try_recv().is_err()); - - state = tokio::task::spawn_blocking(move || { - state.activate_clock(); - state - }) - .await - .unwrap(); - assert!(matches!( - next_event(&mut events).await, - ShadowServiceEventV1::ClockActivated - )); - state.core.shutdown().unwrap(); - } - async fn wait_backlog( events: &mut mpsc::Receiver, expected: (usize, usize, RoundNumber), @@ -5788,7 +2474,6 @@ mod tests { } } - #[allow(clippy::too_many_arguments)] async fn pump_autonomous_until_round( handles: &[StarfishRbcDagShadowServiceHandleV1], events: &mut [mpsc::Receiver], @@ -5799,51 +2484,38 @@ mod tests { sync_requests: &mut usize, projected_vertices: &mut usize, projected_decisions: &mut usize, - max_buffered_authenticated: &mut usize, - rejections: &mut Vec, target_open_round: RoundNumber, - pump_timeout: Duration, ) { - timeout(pump_timeout, async { - let mut pending_network = VecDeque::new(); + timeout(EVENT_TIMEOUT, async { loop { let mut progressed = false; for sender in 0..events.len() { - // Route at most one event per node per pass. Draining one - // sender completely can fill a target actor while that - // target is blocked publishing into its own bounded event - // channel, creating a test-router cycle absent from the - // independent production bridges. - if let Ok(event) = events[sender].try_recv() { + while let Ok(event) = events[sender].try_recv() { progressed = true; match event { ShadowServiceEventV1::Network { recipient, message } => { let recipient = recipient as usize; - let service_message = match message { - NetworkMessage::RbcDagShadowCarrier(envelope) => { - ShadowServiceMessageV1::Carrier { - peer: sender as AuthorityIndex, - envelope, - } - } - NetworkMessage::RbcDagShadowCarrierRequest(reference) => { - ShadowServiceMessageV1::CarrierRequest { - peer: sender as AuthorityIndex, - reference, - } - } - NetworkMessage::RbcDagShadowCarrierResponse(response) => { - ShadowServiceMessageV1::CarrierResponse { - peer: sender as AuthorityIndex, - response, - } - } + match message { + NetworkMessage::RbcDagShadowCarrier(envelope) => handles + [recipient] + .carrier(sender as AuthorityIndex, envelope) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierRequest(reference) => handles + [recipient] + .carrier_request(sender as AuthorityIndex, reference) + .unwrap(), + NetworkMessage::RbcDagShadowCarrierResponse(response) => handles + [recipient] + .carrier_response(sender as AuthorityIndex, response) + .unwrap(), NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { *sync_requests = sync_requests.saturating_add(1); - ShadowServiceMessageV1::CarrierSyncRequest { - peer: sender as AuthorityIndex, - request, - } + handles[recipient] + .carrier_sync_request( + sender as AuthorityIndex, + request, + ) + .unwrap(); } NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { handles[recipient] @@ -5852,34 +2524,14 @@ mod tests { response, ) .unwrap(); - continue; - } - NetworkMessage::RbcDagApplicationPayloadRequest(application) => { - ShadowServiceMessageV1::ApplicationPayloadRequest { - peer: sender as AuthorityIndex, - application, - } - } - NetworkMessage::RbcDagApplicationPayloadResponse(response) => { - ShadowServiceMessageV1::ApplicationPayloadResponse { - peer: sender as AuthorityIndex, - response, - } } unexpected => panic!( "autonomous shadow emitted unexpected network message: {unexpected:?}" ), - }; - pending_network.push_back((recipient, service_message)); + } } - ShadowServiceEventV1::ClockState { - open_round, - buffered_authenticated, - .. - } => { + ShadowServiceEventV1::ClockState { open_round, .. } => { open_rounds[sender] = open_rounds[sender].max(open_round); - *max_buffered_authenticated = - (*max_buffered_authenticated).max(buffered_authenticated); } ShadowServiceEventV1::Delivered(_) => { deliveries[sender] = deliveries[sender].saturating_add(1); @@ -5890,22 +2542,6 @@ mod tests { } => { application_deliveries[sender].insert(header.reference()); } - ShadowServiceEventV1::AuthorizedApplicationObserved { - header, - payload, - .. - } => { - let reference = header.reference(); - if let Some(payload) = payload { - handles[sender] - .verified_application_payload(reference, payload) - .unwrap(); - } - // The production bridge emits availability - // only after the typed header (and optional - // payload) is concretely installed in Core. - handles[sender].application_data_available(reference).unwrap(); - } ShadowServiceEventV1::VertexProjected(_) => { *projected_vertices = projected_vertices.saturating_add(1); } @@ -5916,37 +2552,18 @@ mod tests { committed_frontiers[sender].push(delta); } ShadowServiceEventV1::Rejected { error, .. } - if error.contains("unexpected shadow response") => - { - rejections.push(error); - } + if error.contains("FutureCarrierOutsideBuffer") + || error.contains("unexpected shadow response") => {} ShadowServiceEventV1::Rejected { error, .. } => { - rejections.push(error.clone()); panic!("autonomous shadow rejected valid test traffic: {error}") } _ => {} } } } - let pending = pending_network.len(); - for _ in 0..pending { - let (recipient, message) = pending_network - .pop_front() - .expect("pending network length was captured"); - match handles[recipient].sender.try_send(message) { - Ok(()) => progressed = true, - Err(TrySendError::Full(message)) => { - pending_network.push_back((recipient, message)); - } - Err(TrySendError::Closed(_)) => { - panic!("autonomous shadow target actor stopped") - } - } - } if open_rounds .iter() .all(|round| *round >= target_open_round) - && pending_network.is_empty() { return; } @@ -6012,22 +2629,6 @@ mod tests { ) .unwrap(); } - NetworkMessage::RbcDagApplicationPayloadRequest(application) => { - handles[recipient] - .application_payload_request( - sender as AuthorityIndex, - application, - ) - .unwrap(); - } - NetworkMessage::RbcDagApplicationPayloadResponse(response) => { - handles[recipient] - .application_payload_response( - sender as AuthorityIndex, - response, - ) - .unwrap(); - } unexpected => panic!( "autonomous shadow emitted unexpected network message: {unexpected:?}" ), @@ -6037,19 +2638,6 @@ mod tests { ShadowServiceEventV1::ClockState { open_round, .. } => { open_rounds[sender] = open_rounds[sender].max(open_round); } - ShadowServiceEventV1::AuthorizedApplicationObserved { - header, - payload, - .. - } => { - let reference = header.reference(); - if let Some(payload) = payload { - handles[sender] - .verified_application_payload(reference, payload) - .unwrap(); - } - handles[sender].application_data_available(reference).unwrap(); - } ShadowServiceEventV1::Rejected { error, .. } => { panic!("autonomous shadow rejected valid test traffic: {error}") } @@ -6102,628 +2690,14 @@ mod tests { round, (0..N) .map(|authority| { - *VerifiedBlock::new_genesis(authority as AuthorityIndex).reference() - }) - .collect(), - Vec::new(), - u64::from(round) * 1_000 + u64::from(marker), - TransactionsCommitment::from_bytes([marker; 32]), - ) - .unwrap() - } - - fn application_header_and_payload( - author: AuthorityIndex, - round: RoundNumber, - marker: u8, - committee: &RbcDagCommitteeContextV1, - ) -> (RbcCanonicalHeader, Arc) { - let payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( - Transaction::new(vec![marker; 64]), - )])); - let info_length = committee.committee().info_length(); - let mut encoder = Encoder::new(2, 4, 2).unwrap(); - let encoded = encoder.encode_transactions( - payload.transactions(), - info_length, - committee.committee().len() - info_length, - ); - let commitment = - TransactionsCommitment::new_from_encoded_transactions(&encoded, author as usize).0; - let header = RbcCanonicalHeader::try_new( - author, - round, - committee - .committee() - .authorities() - .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) - .collect(), - Vec::new(), - u64::from(round) * 1_000 + u64::from(marker), - commitment, - ) - .unwrap(); - (header, payload) - } - - fn round_one_application_candidate( - author: AuthorityIndex, - application_header: RbcCanonicalHeader, - committee: &RbcDagCommitteeContextV1, - marker: u8, - ) -> CandidateCarrierV1 { - let weak_parents = committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(carrier_genesis_reference) - .collect(); - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 1, - own_prev: carrier_genesis_reference(author), - weak_parents, - transactions_commitment: application_header.transactions_commitment(), - application_header: Some(application_header), - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: u64::from(marker), - }, - committee, - ) - .unwrap() - } - - #[tokio::test] - async fn far_future_application_is_ignored_before_authentication_or_payload_observation() { - let harness = Harness::new(); - let (header, payload) = application_header_and_payload(0, 1, 0xC0, &harness.committee); - let previous = |authority: AuthorityIndex| BlockReference { - authority, - round: 65, - digest: BlockDigest::from([0xC0 + authority as u8; 32]), - }; - let candidate = CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author: 0, - carrier_round: 66, - own_prev: previous(0), - weak_parents: [1, 2].into_iter().map(previous).collect(), - transactions_commitment: header.transactions_commitment(), - application_header: Some(header), - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: 66, - }, - &harness.committee, - ) - .unwrap(); - let (receiver, mut events, task) = harness.start_autonomous(1); - wait_ready(&mut events).await; - receiver.peer_connected(0).unwrap(); - loop { - if let ShadowServiceEventV1::Network { - recipient: 0, - message: NetworkMessage::RbcDagShadowCarrierSyncRequest(request), - } = next_event(&mut events).await - { - assert_eq!((request.author, request.round), (0, 1)); - break; - } - } - tokio::time::sleep(SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1).await; - receiver - .carrier( - 0, - RbcDagShadowCarrier { - canonical_carrier: candidate.canonical_wire_bytes().unwrap(), - authentication_sidecar: vec![0xff], - application_payload: Some(payload), - }, - ) - .unwrap(); - - let mut ignored = false; - let mut repair_requested = false; - while !ignored || !repair_requested { - match next_event(&mut events).await { - ShadowServiceEventV1::Input { - kind: "carrier", - outcome: "future_ignored", - } => ignored = true, - ShadowServiceEventV1::Network { - recipient: 0, - message: NetworkMessage::RbcDagShadowCarrierSyncRequest(request), - } => { - assert_eq!((request.author, request.round), (0, 1)); - repair_requested = true; - } - ShadowServiceEventV1::AuthorizedApplicationObserved { .. } => { - panic!("far-future application was observed") - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("far-future application was rejected instead of ignored: {error}") - } - _ => {} - } - } - stop(receiver, events, task).await; - } - - #[tokio::test] - async fn initial_payload_fanout_and_authenticated_observation_are_exact() { - let harness = Harness::new(); - let (header, payload) = application_header_and_payload(0, 1, 0xC1, &harness.committee); - - let (author, mut author_events, author_task) = harness.start_autonomous(0); - wait_ready(&mut author_events).await; - author - .local_application(&header, Some(Arc::clone(&payload))) - .unwrap(); - let mut assigned = false; - let mut locally_authorized = false; - let initial = loop { - match next_event(&mut author_events).await { - ShadowServiceEventV1::ApplicationAssigned(reference) - if reference == header.reference() => - { - assigned = true; - } - ShadowServiceEventV1::AuthorizedApplicationObserved { - header: observed, - authorization_basis: ShadowApplicationAuthorizationBasisV1::LocallyFixed, - .. - } if observed == header => { - assert!( - assigned, - "producer gate must be released before materialization" - ); - locally_authorized = true; - } - ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrier(initial), - } => { - assert!(locally_authorized); - break initial; - } - _ => {} - } - }; - assert!(initial.application_payload.is_some()); - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &initial.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - assert_eq!( - candidate - .header() - .application_header() - .map(RbcCanonicalHeader::reference), - Some(header.reference()) - ); - - let (receiver, mut receiver_events, receiver_task) = harness.start_autonomous(1); - wait_ready(&mut receiver_events).await; - receiver.carrier(0, initial).unwrap(); - loop { - match next_event(&mut receiver_events).await { - ShadowServiceEventV1::AuthorizedApplicationObserved { - carrier, - header: observed, - payload: Some(observed_payload), - authorization_basis: - ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, - } => { - assert_eq!(carrier, candidate.reference()); - assert_eq!(observed, header); - assert!(application_payloads_equal( - Some(observed_payload.as_ref()), - Some(payload.as_ref()) - )); - break; - } - ShadowServiceEventV1::Delivered(_) => { - panic!("fresh receiver authentication should stage before delivery") - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("valid authenticated application was rejected: {error}") - } - _ => {} - } - } - stop(author, author_events, author_task).await; - stop(receiver, receiver_events, receiver_task).await; - } - - #[tokio::test] - async fn authorized_empty_application_never_requests_payload_across_retries() { - let harness = Harness::new(); - let header = RbcCanonicalHeader::try_new( - 0, - 1, - harness - .committee - .committee() - .authorities() - .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) - .collect(), - Vec::new(), - 1_001, - TransactionsCommitment::default(), - ) - .unwrap(); - let candidate = - round_one_application_candidate(0, header.clone(), &harness.committee, 0xC4); - - let (receiver, mut events, task) = harness.start_autonomous(1); - wait_ready(&mut events).await; - receiver.peer_connected(0).unwrap(); - receiver - .carrier(0, harness.envelope(&candidate, 0)) - .unwrap(); - - let mut authorized = false; - let deadline = - Instant::now() + SHADOW_APPLICATION_PAYLOAD_RETRY_INTERVAL_V1.saturating_mul(3); - while Instant::now() < deadline { - let remaining = deadline.saturating_duration_since(Instant::now()); - let event = match timeout(remaining, events.recv()).await { - Ok(Some(event)) => event, - Ok(None) => panic!("shadow actor stopped while checking empty application"), - Err(_) => break, - }; - match event { - ShadowServiceEventV1::AuthorizedApplicationObserved { - header: observed, - payload: None, - authorization_basis: - ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, - .. - } if observed == header => { - authorized = true; - receiver - .application_data_available(header.reference()) - .expect("empty authorized application must accept the DA callback"); - } - ShadowServiceEventV1::Network { - message: NetworkMessage::RbcDagApplicationPayloadRequest(application), - .. - } if application == header.reference() => { - panic!("empty authorized application must not request payload bytes") - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("empty authorized application was rejected: {error}") - } - _ => {} - } - } - assert!(authorized, "empty application header was not authorized"); - stop(receiver, events, task).await; - } - - #[tokio::test] - async fn authorized_payload_recovery_is_peer_bound_verified_and_coalesced() { - let harness = Harness::new(); - let (header, payload) = application_header_and_payload(0, 1, 0xC4, &harness.committee); - let candidate = - round_one_application_candidate(0, header.clone(), &harness.committee, 0xC5); - let envelope = harness.envelope(&candidate, 0); - - let (receiver, mut events, task) = harness.start_autonomous(1); - wait_ready(&mut events).await; - receiver.peer_connected(0).unwrap(); - receiver.peer_connected(2).unwrap(); - receiver.carrier(0, envelope).unwrap(); - - let mut observed_header = false; - let mut requested = false; - while !observed_header || !requested { - match next_event(&mut events).await { - ShadowServiceEventV1::AuthorizedApplicationObserved { - header: observed, - payload: None, - authorization_basis: - ShadowApplicationAuthorizationBasisV1::ReceiverAuthenticated, - .. - } if observed == header => observed_header = true, - ShadowServiceEventV1::Network { - recipient: 0, - message: NetworkMessage::RbcDagApplicationPayloadRequest(application), - } if application == header.reference() => requested = true, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("authorized header-only carrier was rejected: {error}") - } - _ => {} - } - } - - receiver - .application_payload_response( - 2, - RbcDagApplicationPayloadResponse { - application: header.reference(), - transaction_data: Arc::clone(&payload), - }, - ) - .unwrap(); - loop { - if let ShadowServiceEventV1::Rejected { - peer: Some(2), - error, - } = next_event(&mut events).await - { - assert!(error.contains("unrequested authority")); - break; - } - } - - receiver - .application_payload_response( - 0, - RbcDagApplicationPayloadResponse { - application: header.reference(), - transaction_data: Arc::clone(&payload), - }, - ) - .unwrap(); - loop { - if let ShadowServiceEventV1::AuthorizedApplicationObserved { - header: observed, - payload: Some(observed_payload), - .. - } = next_event(&mut events).await - { - assert_eq!(observed, header); - assert!(application_payloads_equal( - Some(observed_payload.as_ref()), - Some(payload.as_ref()) - )); - break; - } - } - - receiver - .application_payload_response( - 0, - RbcDagApplicationPayloadResponse { - application: header.reference(), - transaction_data: Arc::clone(&payload), - }, - ) - .unwrap(); - receiver - .verified_application_payload(header.reference(), Arc::clone(&payload)) - .unwrap(); - receiver - .application_payload_request(2, header.reference()) - .unwrap(); - loop { - match next_event(&mut events).await { - ShadowServiceEventV1::Network { - recipient: 2, - message: NetworkMessage::RbcDagApplicationPayloadResponse(response), - } => { - assert_eq!(response.application, header.reference()); - assert!(application_payloads_equal( - Some(response.transaction_data.as_ref()), - Some(payload.as_ref()) - )); - break; - } - ShadowServiceEventV1::AuthorizedApplicationObserved { - payload: Some(_), .. - } => panic!("duplicate payload response emitted duplicate application work"), - _ => {} - } - } - receiver - .application_payload_request(2, header.reference()) - .unwrap(); - loop { - if let ShadowServiceEventV1::Input { - kind: "application_payload_request", - outcome: "rate_limited", - } = next_event(&mut events).await - { - break; - } - } - stop(receiver, events, task).await; - } - - #[tokio::test] - async fn application_and_quarantine_maps_remain_bounded() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (state, _events, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - tokio::task::spawn_blocking(move || { - let mut state = state; - for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - let round = offset as RoundNumber + 1; - let header = direct_header(0, round, offset as u8); - state - .authorize_application( - BlockReference::new_test(0, 1_000 + round), - header, - None, - false, - None, - ShadowApplicationAuthorizationBasisV1::Delivered, - ) - .unwrap(); - } - assert_eq!( - state.authorized_applications.len(), - SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 - ); - - for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - let round = offset as RoundNumber + 1; - state - .quarantine_application( - BlockReference::new_test(1, 2_000 + round), - BlockReference::new_test(1, round), - 1, - None, - ) - .unwrap(); - } - assert_eq!( - state.quarantined_application_payloads.len(), - SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 - ); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - } - - #[tokio::test] - async fn delayed_verified_payload_for_evicted_application_is_stale_not_rejected() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (state, mut events, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - tokio::task::spawn_blocking(move || { - let mut state = state; - let mut applications = Vec::new(); - for offset in 0..=SHADOW_APPLICATION_PAYLOAD_CAPACITY_V1 { - let round = offset as RoundNumber + 1; - let header = direct_header(0, round, offset as u8); - applications.push(header.reference()); - state - .authorize_application( - BlockReference::new_test(0, 3_000 + round), - header, - None, - false, - None, - ShadowApplicationAuthorizationBasisV1::Delivered, - ) - .unwrap(); - } - let evicted = applications - .into_iter() - .find(|application| !state.authorized_applications.contains_key(application)) - .expect("one authorized application must be evicted at capacity plus one"); - let delayed_payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( - Transaction::new(vec![0xD3; 64]), - )])); - state - .desired_verified_application_payloads - .lock() - .insert(evicted, delayed_payload); - state.reconcile_verified_application_payloads(); - assert!( - state - .desired_verified_application_payloads - .lock() - .is_empty() - ); - assert!(!state.fatal); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - - let mut observed_stale = false; - while let Ok(event) = events.try_recv() { - match event { - ShadowServiceEventV1::Input { - kind: "verified_application_payload", - outcome: "stale_ignored", - } => observed_stale = true, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("delayed verified callback was rejected: {error}") - } - _ => {} - } - } - assert!(observed_stale, "stale callback outcome was not reported"); - } - - #[tokio::test] - async fn delayed_payload_response_for_evicted_application_is_stale_not_rejected() { - let harness = Harness::new(); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (state, mut events, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - tokio::task::spawn_blocking(move || { - let mut state = state; - let evicted = BlockReference::new_test(1, 1); - let payload = Arc::new(TransactionData::new(vec![BaseTransaction::Share( - Transaction::new(vec![0xD4; 64]), - )])); - state - .handle_application_payload_response( - 1, - RbcDagApplicationPayloadResponse { - application: evicted, - transaction_data: payload, - }, - ) - .expect("late network completion must be an idempotent stale observation"); - assert!(!state.fatal); - state.core.shutdown().unwrap(); - }) - .await - .unwrap(); - - let mut observed_stale = false; - while let Ok(event) = events.try_recv() { - match event { - ShadowServiceEventV1::Input { - kind: "application_payload_response", - outcome: "stale_ignored", - } => observed_stale = true, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("delayed payload response was rejected: {error}") - } - _ => {} - } - } - assert!( - observed_stale, - "stale network response outcome was not reported" - ); + *VerifiedBlock::new_genesis(authority as AuthorityIndex).reference() + }) + .collect(), + Vec::new(), + u64::from(round) * 1_000 + u64::from(marker), + TransactionsCommitment::from_bytes([marker; 32]), + ) + .unwrap() } fn round_one_candidate( @@ -6808,244 +2782,13 @@ mod tests { stop(handle, events, task).await; } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn instrumented_autonomous_service_records_assignment_and_queue_state() { - let harness = Harness::new(); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new(®istry, None, None, None); - metrics.metrics_active.store(true, Ordering::Relaxed); - let (handle, mut events, task) = - start_starfish_rbc_dag_autonomous_clock_service_with_metrics_v1( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - Vec::new(), - Duration::from_secs(60 * 60), - ShadowWalSyncPolicyV1::EveryBatch, - Arc::clone(&metrics), - ) - .unwrap(); - wait_ready(&mut events).await; - - let application = direct_header(0, 1, 0xA1); - handle.local_header(&application).unwrap(); - let mut application_assigned = false; - timeout(EVENT_TIMEOUT, async { - loop { - match events.recv().await { - Some(ShadowServiceEventV1::ApplicationAssigned(reference)) => { - application_assigned = true; - assert_eq!(reference, application.reference()); - } - Some(ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrier(_), - }) => { - assert!( - application_assigned, - "the local producer gate must be released before network fan-out" - ); - break; - } - _ => {} - } - } - }) - .await - .expect("carrier transition did not publish its assignment and network events"); - assert!(application_assigned); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_latency_samples_total - .with_label_values(&[RBC_DAG_LATENCY_CREATION_TO_ASSIGNMENT]) - .get(), - 1 - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_queue_depth - .with_label_values(&["local"]) - .get(), - 0 - ); - assert_eq!( - metrics - .starfish_rbc_dag_pipeline_queue_depth_max - .with_label_values(&["local"]) - .get(), - 1 - ); - assert_eq!( - metrics - .starfish_rbc_dag_projection_hol_state - .with_label_values(&["insufficient_lookahead"]) - .get(), - 1 - ); - - stop(handle, events, task).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn recovered_application_assignment_does_not_release_a_live_producer_gate() { - let harness = Harness::new(); - let application = direct_header(0, 1, 0xA2); - let (handle, mut events, task) = start_starfish_rbc_dag_autonomous_clock_service_v1( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - vec![application], - Duration::from_secs(60 * 60), - ShadowWalSyncPolicyV1::EveryBatch, - ) - .unwrap(); - - timeout(EVENT_TIMEOUT, async { - loop { - match events.recv().await { - Some(ShadowServiceEventV1::ApplicationAssigned(reference)) => panic!( - "recovered application {reference} has no live producer gate to release" - ), - Some(ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrier(_), - }) => break, - Some(ShadowServiceEventV1::Rejected { error, .. }) => { - panic!("recovered application was rejected: {error}") - } - _ => {} - } - } - }) - .await - .expect("recovered application was not assigned to the open carrier"); - - stop(handle, events, task).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn live_duplicate_upgrades_recovered_pending_application_to_exactly_one_ack() { - let harness = Harness::new(); - let application = direct_header(0, 1, 0xA3); - let (core, _) = StarfishRbcDagShadowV1::open( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - ) - .unwrap(); - let (mut state, mut events, _message_rx, _message_tx) = standalone_autonomous_state( - core, - harness.committee.clone(), - Duration::from_secs(60 * 60), - ); - state.pending_local.insert( - application.reference().round, - ShadowLocalCarrierV1::from_recovered_direct_header(&application), - ); - - let expected_reference = application.reference(); - let state = tokio::task::spawn_blocking(move || { - state.enqueue_local(ShadowLocalCarrierV1::from_direct_header(&application)); - assert!( - state - .pending_local - .get(&application.reference().round) - .is_some_and(|pending| pending.acknowledge_assignment) - ); - state.try_create_autonomous_carrier(); - state - }) - .await - .unwrap(); - let acknowledgments = std::iter::from_fn(|| events.try_recv().ok()) - .filter_map(|event| match event { - ShadowServiceEventV1::ApplicationAssigned(reference) => Some(reference), - _ => None, - }) - .collect::>(); - assert_eq!(acknowledgments, vec![expected_reference]); - state.core.shutdown().unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn paused_autonomous_clock_uses_one_idempotent_fresh_activation_epoch() { - let harness = Harness::new(); - let heartbeat_interval = Duration::from_millis(120); - let (handle, mut events, task) = - harness.start_autonomous_paused_with_interval(0, heartbeat_interval); - wait_ready(&mut events).await; - - // Neither the dormant timer nor an explicitly injected stale tick may - // fix a carrier while the external coordinated-start barrier is shut. - tokio::time::sleep(heartbeat_interval.saturating_mul(4)).await; - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - tokio::time::sleep(Duration::from_millis(20)).await; - let inspection = handle.inspect_carrier_sync().await.unwrap(); - assert_eq!(inspection.open_round, 1); - while let Ok(event) = events.try_recv() { - assert!(!matches!(event, ShadowServiceEventV1::ClockActivated)); - assert!( - !matches!(event, ShadowServiceEventV1::Network { .. }), - "paused autonomous clock emitted protocol network traffic: {event:?}" - ); - } - - let activated_at = Instant::now(); - handle.activate_clock().await.unwrap(); - handle.activate_clock().await.unwrap(); - - let mut activation_events = 0; - let early_network = timeout(heartbeat_interval / 2, async { - loop { - match events.recv().await { - Some(ShadowServiceEventV1::ClockActivated) => activation_events += 1, - Some(ShadowServiceEventV1::Network { .. }) => return, - Some(ShadowServiceEventV1::Rejected { error, .. }) => { - panic!("activated clock rejected input: {error}") - } - Some(_) => {} - None => panic!("activated clock stopped before its first heartbeat"), - } - } - }) - .await; - assert!( - early_network.is_err(), - "activation inherited a stale heartbeat deadline" - ); - assert_eq!(activation_events, 1, "activation was not idempotent"); - - timeout( - heartbeat_interval.saturating_mul(3), - next_carrier(&mut events, 1), - ) - .await - .expect("fresh activation epoch did not produce its first heartbeat"); - assert!( - activated_at.elapsed() >= heartbeat_interval.saturating_sub(Duration::from_millis(20)), - "first heartbeat did not wait one fresh interval" - ); - - stop(handle, events, task).await; - } - async fn assert_autonomous_zero_load_progress(n: usize) { let harness = Harness::new_with_n(n); let mut handles = Vec::new(); let mut events = Vec::new(); let mut tasks = Vec::new(); for authority in 0..n as AuthorityIndex { - // Exercise the production 600 ms clock so the shared 30 ms - // normal-carrier permit advances under real wall time. The test - // still injects heartbeat triggers explicitly for determinism. - let (handle, mut node_events, task) = - harness.start_autonomous_with_interval(authority, Duration::from_millis(600)); + let (handle, mut node_events, task) = harness.start_autonomous(authority); loop { match next_event(&mut node_events).await { ShadowServiceEventV1::Ready { autonomous_clock } => { @@ -7070,8 +2813,6 @@ mod tests { let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; - let mut max_buffered_authenticated = 0; - let mut rejections = Vec::new(); for (authority, handle) in handles.iter().enumerate() { for peer in 0..n { if peer != authority { @@ -7093,26 +2834,9 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, fixed_round + 1, - EVENT_TIMEOUT, ) .await; - if fixed_round == 2 { - for (authority, handle) in handles.iter().enumerate() { - let (optimistic_promises, certified_deliveries) = - handle.inspect_rbc_progress().await.unwrap(); - assert!( - optimistic_promises > 0, - "authority {authority} did not lock an optimistic ECHO promise by the round-two carrier" - ); - assert_eq!( - certified_deliveries, 0, - "authority {authority} certified a carrier before the later READY round" - ); - } - } } assert!( @@ -7132,9 +2856,8 @@ mod tests { "clean projection did not decide: vertices={projected_vertices}, rounds={open_rounds:?}" ); assert_eq!( - sync_requests, - n * (n - 1), - "each connection must repair exactly the round-one carrier fixed before topology registration" + sync_requests, 0, + "healthy proactive rounds must not trigger repair polling" ); drop(events); @@ -7179,8 +2902,7 @@ mod tests { let mut events = Vec::new(); let mut tasks = Vec::new(); for authority in 0..N as AuthorityIndex { - let (handle, mut node_events, task) = - harness.start_autonomous_with_interval(authority, Duration::from_millis(600)); + let (handle, mut node_events, task) = harness.start_autonomous(authority); wait_ready(&mut node_events).await; handles.push(handle); events.push(node_events); @@ -7215,8 +2937,6 @@ mod tests { let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; - let mut max_buffered_authenticated = 0; - let mut rejections = Vec::new(); pump_autonomous_until_round( &handles, &mut events, @@ -7227,16 +2947,13 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, 5, - EVENT_TIMEOUT, ) .await; // The first directly committed consensus frontier may predate the // round-one application deliveries. Advance enough certified carrier // rounds for a later committed frontier to include that closed prefix. - for fixed_round in 5..=40 { + for fixed_round in 5..=18 { for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -7250,21 +2967,9 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, fixed_round + 1, - EVENT_TIMEOUT, ) .await; - if committed_frontiers.iter().all(|commits| { - commits - .iter() - .flat_map(|delta| delta.applications.iter().map(RbcCanonicalHeader::reference)) - .collect::>() - == expected - }) { - break; - } } assert!( @@ -7277,11 +2982,7 @@ mod tests { open_rounds.iter().all(|round| *round >= 5), "the carrier DAG must keep advancing after application delivery" ); - assert_eq!( - sync_requests, - N * (N - 1), - "each connection must repair exactly the round-one carrier fixed before topology registration" - ); + assert_eq!(sync_requests, 0); assert!( projected_vertices >= N, "empty embedded applications must not stall clean projection" @@ -7433,17 +3134,13 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn pipelined_exact_sync_closes_gap_beyond_retention_while_producers_run() { + async fn autonomous_exact_sync_closes_a_multi_round_gap() { let harness = Harness::new(); let mut handles = Vec::new(); let mut events = Vec::new(); let mut tasks = Vec::new(); for authority in 0..N as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous_with_policy( - authority, - Duration::from_millis(600), - ShadowWalSyncPolicyV1::OnShutdown, - ); + let (handle, mut node_events, task) = harness.start_autonomous(authority); wait_ready(&mut node_events).await; handles.push(handle); events.push(node_events); @@ -7453,9 +3150,8 @@ mod tests { // Establish round one for all validators, then let a quorum advance // while authority 3 is offline and receives none of the proactive // carriers. Starting the gap at round two makes reconnect request - // exact repair immediately. The normal pacer drives the healthy - // prefix, while the later lagger convergence must use the bounded - // validity-backed repair lane rather than wait for normal permits. + // exact repair immediately; the one-hour normal heartbeat still + // cannot help with the later repaired rounds. for (authority, handle) in handles.iter().enumerate() { for peer in 0..N { if peer != authority { @@ -7470,8 +3166,6 @@ mod tests { let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; - let mut max_buffered_authenticated = 0; - let mut rejections = Vec::new(); for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -7485,10 +3179,7 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, 2, - EVENT_TIMEOUT, ) .await; for authority in 0..3 { @@ -7497,7 +3188,7 @@ mod tests { .peer_disconnected(authority as AuthorityIndex) .unwrap(); } - for fixed_round in 2..=72 { + for fixed_round in 2..=8 { for handle in &handles[..3] { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -7511,12 +3202,12 @@ mod tests { .await; } assert_eq!(open_rounds[3], 2); - assert!(open_rounds[..3].iter().all(|round| *round >= 73)); + assert!(open_rounds[..3].iter().all(|round| *round >= 9)); // Reconnect, fix the lagging node's first local slot, and let the // healthy quorum open one more round. Exact responses then drive an - // immediate local heartbeat per repaired round; convergence must - // outrun the independently paced healthy producers. + // immediate local heartbeat per repaired round; the one-hour normal + // timer cannot be responsible for convergence. for (authority, handle) in handles.iter().enumerate() { for peer in 0..N { if peer != authority { @@ -7527,26 +3218,9 @@ mod tests { handles[3] .send(ShadowServiceMessageV1::HeartbeatTick) .unwrap(); - let catch_up_initial_gap = open_rounds[..3] - .iter() - .copied() - .min() - .unwrap() - .saturating_sub(open_rounds[3]); - let producer_running = Arc::new(AtomicBool::new(true)); - let producer_flag = Arc::clone(&producer_running); - let producer_handles = handles[..3].to_vec(); - let producer = tokio::spawn(async move { - while producer_flag.load(Ordering::Relaxed) { - for handle in &producer_handles { - handle - .send_reliably(ShadowServiceMessageV1::HeartbeatTick) - .await - .unwrap(); - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }); + for handle in &handles[..3] { + handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); + } let sync_requests_before_catch_up = sync_requests; pump_autonomous_until_round( @@ -7559,213 +3233,15 @@ mod tests { &mut sync_requests, &mut projected_vertices, &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, - 80, - Duration::from_secs(30), + 10, ) .await; - let inspect_handle = handles[3].clone(); - let inspection_task = tokio::spawn(async move { - inspect_handle - .inspect_carrier_sync() - .await - .expect("lagging actor stopped before live catch-up inspection") - }); - timeout(Duration::from_secs(5), async { - while !inspection_task.is_finished() { - let observed_minimum = open_rounds.iter().copied().min().unwrap_or_default(); - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut application_deliveries, - &mut committed_frontiers, - &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, - observed_minimum, - Duration::from_secs(1), - ) - .await; - tokio::task::yield_now().await; - } - }) - .await - .expect("live catch-up inspection was starved behind actor events"); - let live_inspection = inspection_task.await.unwrap(); - let live_lagger_round = open_rounds[3].max(live_inspection.open_round); - let live_healthy_minimum = open_rounds[..3].iter().copied().min().unwrap(); - // A fast run may already have reached and drained the validity-backed - // target, in which case the actor correctly clears the episode. Treat - // that stronger state as a zero target gap at the inspected round. - let live_target = live_inspection.target.unwrap_or(live_lagger_round); - let live_healthy_gap = live_healthy_minimum.saturating_sub(live_lagger_round); - let live_target_gap = live_target.saturating_sub(live_lagger_round); - assert!( - live_lagger_round.saturating_sub(2) > EXECUTABLE_MODEL_BUFFER_WINDOW_V1, - "lagger did not repair beyond the normal retention horizon: live={open_rounds:?}, inspection={live_inspection:?}" - ); - assert!( - live_healthy_gap < catch_up_initial_gap, - "lagger gap did not shrink while producers remained live: initial_gap={catch_up_initial_gap}, live={open_rounds:?}, inspection={live_inspection:?}" - ); - assert!( - live_target_gap <= live_healthy_gap, - "validity-backed target escaped beyond the healthy live high-water: live={open_rounds:?}, inspection={live_inspection:?}" - ); - - // A single shrinking observation is not sufficient: repair could - // still plateau permanently outside the bounded exact-slot window. - // Keep the producers live and route fairly until the lagger is within - // one pipeline of both the validity-backed target and the observed - // healthy minimum. A fully drained episode that already reached and - // cleared its target is the stronger equivalent outcome. - let pipeline_depth = carrier_sync_pipeline_depth(N) as RoundNumber; - let (bounded_inspection, bounded_lagger_round) = - timeout(Duration::from_secs(5), async { - loop { - let inspect_handle = handles[3].clone(); - let inspection_task = tokio::spawn(async move { - inspect_handle.inspect_carrier_sync().await.expect( - "lagging actor stopped before bounded catch-up inspection", - ) - }); - while !inspection_task.is_finished() { - let observed_minimum = - open_rounds.iter().copied().min().unwrap_or_default(); - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut application_deliveries, - &mut committed_frontiers, - &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, - observed_minimum, - Duration::from_secs(1), - ) - .await; - tokio::task::yield_now().await; - } - let inspection = inspection_task.await.unwrap(); - let lagger_round = open_rounds[3].max(inspection.open_round); - let healthy_minimum = open_rounds[..3].iter().copied().min().unwrap(); - let healthy_gap = healthy_minimum.saturating_sub(lagger_round); - let target_gap = inspection - .target - .map(|target| target.saturating_sub(lagger_round)) - .unwrap_or_default(); - let episode_cleared = inspection.target.is_none() - && inspection.outstanding == 0 - && inspection.desired_responses == 0; - if episode_cleared - || (healthy_gap <= pipeline_depth && target_gap <= pipeline_depth) - { - break (inspection, lagger_round); - } - } - }) - .await - .unwrap_or_else(|_| { - panic!( - "live exact repair plateaued outside pipeline depth {pipeline_depth}: open={open_rounds:?}, first_inspection={live_inspection:?}" - ) - }); - producer_running.store(false, Ordering::Relaxed); - producer.await.unwrap(); assert!( sync_requests > sync_requests_before_catch_up, "catch-up must use exact-slot repair" ); - // Maintenance timers may keep publishing benign state observations, - // so quiescence cannot mean an indefinitely empty event channel. - // Fairly route one event per node for one complete exact-slot budget; - // every staged network event is delivered before each pass returns. - for _ in 0..SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1 { - let observed_minimum = open_rounds.iter().copied().min().unwrap_or_default(); - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut application_deliveries, - &mut committed_frontiers, - &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, - observed_minimum, - Duration::from_secs(1), - ) - .await; - } - let inspect_handle = handles[3].clone(); - let final_inspection_task = tokio::spawn(async move { - inspect_handle - .inspect_carrier_sync() - .await - .expect("lagging actor stopped before final catch-up inspection") - }); - timeout(Duration::from_secs(2), async { - while !final_inspection_task.is_finished() { - let observed_minimum = open_rounds.iter().copied().min().unwrap_or_default(); - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut application_deliveries, - &mut committed_frontiers, - &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, - &mut max_buffered_authenticated, - &mut rejections, - observed_minimum, - Duration::from_secs(1), - ) - .await; - tokio::task::yield_now().await; - } - }) - .await - .expect("final catch-up inspection timed out behind actor events"); - let final_inspection = final_inspection_task.await.unwrap(); - assert!( - final_inspection.open_round >= bounded_lagger_round, - "short router drain regressed catch-up progress: bounded={bounded_inspection:?}, final={final_inspection:?}" - ); - assert!(max_buffered_authenticated > 0); - let buffered_authenticated_capacity = usize::try_from( - EXECUTABLE_MODEL_BUFFER_WINDOW_V1.saturating_sub(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1), - ) - .unwrap_or(usize::MAX) - .saturating_mul(N.saturating_sub(1)); - assert!( - max_buffered_authenticated <= buffered_authenticated_capacity, - "pipelined exact repair exceeded the executable model's authenticated retention capacity: observed={max_buffered_authenticated}, capacity={buffered_authenticated_capacity}" - ); - assert!( - final_inspection.max_outstanding <= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, - "outstanding exact slots exceeded the global cap: {final_inspection:?}" - ); - assert!( - final_inspection.max_desired_responses <= SHADOW_CARRIER_SYNC_SLOT_CAPACITY_V1, - "coalesced exact responses exceeded the global cap: {final_inspection:?}" - ); - assert!( - rejections.is_empty(), - "valid catch-up produced rejections: {rejections:?}" - ); + assert!(open_rounds.iter().all(|round| *round >= 10)); + drop(events); for handle in &handles { handle.shutdown().await.unwrap(); @@ -8120,79 +3596,6 @@ mod tests { stop(receiver, receiver_events, receiver_task).await; } - #[tokio::test] - async fn poisoned_application_payload_waits_for_exact_delivery() { - let harness = Harness::new(); - let (header, payload) = application_header_and_payload(3, 1, 0xC2, &harness.committee); - let target = round_one_application_candidate(3, header.clone(), &harness.committee, 0xC3); - let mut envelope = harness.envelope(&target, 3); - envelope.application_payload = Some(Arc::clone(&payload)); - envelope.authentication_sidecar[3 + 2 * MAC_TAG_SIZE] ^= 1; - - let (receiver, mut events, task) = harness.start(2, Vec::new()); - wait_ready(&mut events).await; - receiver.carrier(3, envelope).unwrap(); - let mut retained = false; - let mut rejected = false; - while !retained || !rejected { - match next_event(&mut events).await { - ShadowServiceEventV1::Input { - kind: "carrier", - outcome: "retained_unauthenticated", - } => retained = true, - ShadowServiceEventV1::Rejected { peer: Some(3), .. } => rejected = true, - ShadowServiceEventV1::AuthorizedApplicationObserved { .. } => { - panic!("poisoned receiver MAC must not authorize its payload") - } - _ => {} - } - } - - for author in [0, 1] { - let phase = phase_carrier( - author, - RbcPhaseStatementV1::Ready { - target: target.reference(), - }, - &harness.committee, - ); - receiver - .carrier(author, harness.envelope(&phase, author)) - .unwrap(); - } - let mut delivered = false; - loop { - match next_event(&mut events).await { - ShadowServiceEventV1::Delivered(identity) - if identity.author == 3 && identity.round == 1 => - { - delivered = true; - } - ShadowServiceEventV1::AuthorizedApplicationObserved { - carrier, - header: observed, - payload: Some(observed_payload), - authorization_basis: ShadowApplicationAuthorizationBasisV1::Delivered, - } => { - assert!(delivered, "authorization must follow the Delivered effect"); - assert_eq!(carrier, target.reference()); - assert_eq!(observed, header); - assert!(application_payloads_equal( - Some(observed_payload.as_ref()), - Some(payload.as_ref()) - )); - break; - } - ShadowServiceEventV1::Rejected { - peer: Some(peer), - error, - } if peer != 3 => panic!("delivery evidence was rejected: {error}"), - _ => {} - } - } - stop(receiver, events, task).await; - } - #[tokio::test] async fn reconnect_replays_exact_persisted_envelope() { let harness = Harness::new(); @@ -8409,67 +3812,6 @@ mod tests { stop(handle, events, task).await; } - #[test] - fn verified_payload_callback_coalesces_when_notification_queue_is_full() { - let harness = Harness::new(); - let (header, payload) = application_header_and_payload(0, 1, 0xD1, &harness.committee); - let (other_header, other_payload) = - application_header_and_payload(0, 2, 0xD2, &harness.committee); - let (sender, mut receiver) = mpsc::channel(1); - let desired_verified_application_payloads = Arc::new(Mutex::new(BTreeMap::new())); - let invalidated_by_overload = Arc::new(Mutex::new(None)); - let handle = StarfishRbcDagShadowServiceHandleV1 { - sender, - mode: ShadowServiceModeV1::DirectMirror, - max_sidecar_size: 3 + N * MAC_TAG_SIZE, - own_authority: 0, - committee_size: N, - input_capacity: 1, - desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), - desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), - desired_verified_application_payloads: Arc::clone( - &desired_verified_application_payloads, - ), - desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), - invalidated_by_overload: Arc::clone(&invalidated_by_overload), - }; - - handle.send(ShadowServiceMessageV1::RetryRecovery).unwrap(); - handle - .verified_application_payload(header.reference(), Arc::clone(&payload)) - .unwrap(); - handle - .verified_application_payload(header.reference(), Arc::clone(&payload)) - .unwrap(); - - assert_eq!(desired_verified_application_payloads.lock().len(), 1); - assert_eq!(*invalidated_by_overload.lock(), None); - assert!(matches!( - receiver.try_recv(), - Ok(ShadowServiceMessageV1::RetryRecovery) - )); - assert!(matches!( - receiver.try_recv(), - Err(mpsc::error::TryRecvError::Empty) - )); - - handle - .verified_application_payload(other_header.reference(), Arc::clone(&other_payload)) - .unwrap(); - assert!(matches!( - receiver.try_recv(), - Ok(ShadowServiceMessageV1::VerifiedApplicationPayloadsChanged) - )); - assert_eq!(desired_verified_application_payloads.lock().len(), 2); - assert!(matches!( - handle.verified_application_payload(header.reference(), other_payload), - Err(ShadowServiceErrorV1::ConflictingApplicationPayload(application)) - if application == header.reference() - )); - } - #[tokio::test] async fn bounded_input_reports_overload_but_shutdown_waits_for_capacity() { let input_capacity = shadow_input_capacity(N, ShadowServiceModeV1::DirectMirror).unwrap(); @@ -8482,9 +3824,6 @@ mod tests { committee_size: N, input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), - desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), - desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), @@ -8496,7 +3835,6 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![round as u8], authentication_sidecar: Vec::new(), - application_payload: None, }, ) .unwrap(); @@ -8507,7 +3845,6 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![0xFF], authentication_sidecar: Vec::new(), - application_payload: None, }, ), Err(ShadowServiceErrorV1::Overloaded { @@ -8538,9 +3875,6 @@ mod tests { committee_size: N, input_capacity: 1, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), - desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), - desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), @@ -8551,7 +3885,6 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![0; MAX_CARRIER_CONTENT_SIZE_V1 + 1], authentication_sidecar: Vec::new(), - application_payload: None, }, ), Err(ShadowServiceErrorV1::InputTooLarge { @@ -8577,9 +3910,6 @@ mod tests { committee_size: LARGE_N, input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_local_applications: Arc::new(Mutex::new(BTreeMap::new())), - desired_carrier_sync_responses: Arc::new(Mutex::new(BTreeMap::new())), - desired_verified_application_payloads: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::clone(&invalidated), @@ -8591,7 +3921,6 @@ mod tests { RbcDagShadowCarrier { canonical_carrier: vec![peer as u8], authentication_sidecar: Vec::new(), - application_payload: None, }, ) .unwrap(); diff --git a/crates/starfish-core/src/stat.rs b/crates/starfish-core/src/stat.rs index c18b81fe..34f6b540 100644 --- a/crates/starfish-core/src/stat.rs +++ b/crates/starfish-core/src/stat.rs @@ -94,16 +94,6 @@ impl PreciseHistogram { self.points.clear(); } - /// Reset both the current distribution and its lifetime aggregates. - /// Used only when a benchmark establishes a new explicit observation - /// epoch; periodic reporting continues to use `clear` and preserves its - /// historical running totals. - pub fn reset(&mut self) { - self.points.clear(); - self.sum = T::default(); - self.count = 0; - } - fn pct1000_index(&self, pct1000: usize) -> usize { debug_assert!(pct1000 < 1000); self.points.len() * pct1000 / 1000 diff --git a/crates/starfish-core/src/store.rs b/crates/starfish-core/src/store.rs index 32607a20..31ad4fbd 100644 --- a/crates/starfish-core/src/store.rs +++ b/crates/starfish-core/src/store.rs @@ -3,208 +3,12 @@ use std::io; -use serde::{Deserialize, Serialize}; - use crate::{ - crypto::BLOCK_DIGEST_SIZE, dag_state::CommitData, data::Data, - types::{BlockReference, MAX_COMMITTEE_SIZE, ProvableShard, RoundNumber, VerifiedBlock}, + types::{BlockReference, ProvableShard, RoundNumber, VerifiedBlock}, }; -/// Durable acknowledgement that Core applied one RBC-DAG committed frontier. -/// -/// This is a single, bounded latest-value cursor rather than an append-only -/// history. `carrier_anchor` is intentionally independent of application block -/// storage: an RBC-DAG consensus carrier is not necessarily a Core block. The -/// per-authority watermark lets Core restore commit progress without scanning -/// application references. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct RbcDagFrontierReceipt { - pub(crate) carrier_anchor: BlockReference, - /// Monotone one-based position in the authoritative frontier output - /// stream. This is intentionally independent of the anchor's logical - /// consensus round, which may regress when a later certifier resolves an - /// older leader. - pub(crate) output_sequence: RoundNumber, - pub(crate) committed_rounds: Vec, -} - -impl RbcDagFrontierReceipt { - const ENCODING_MAGIC: [u8; 4] = *b"RDF1"; - const FIXED_ENCODED_LEN: usize = Self::ENCODING_MAGIC.len() - + std::mem::size_of::() - + std::mem::size_of::() - + BLOCK_DIGEST_SIZE - + std::mem::size_of::() - + std::mem::size_of::(); - const MAX_ENCODED_LEN: usize = - Self::FIXED_ENCODED_LEN + MAX_COMMITTEE_SIZE as usize * std::mem::size_of::(); - - fn validate(&self, error_kind: io::ErrorKind) -> io::Result<()> { - if self.output_sequence == 0 { - return Err(io::Error::new( - error_kind, - "RBC-DAG frontier receipt output sequence must be nonzero", - )); - } - if self.committed_rounds.is_empty() - || self.committed_rounds.len() > usize::from(MAX_COMMITTEE_SIZE) - { - return Err(io::Error::new( - error_kind, - format!( - "RBC-DAG frontier receipt must contain 1..={MAX_COMMITTEE_SIZE} committed-round watermarks, got {}", - self.committed_rounds.len() - ), - )); - } - if self.carrier_anchor.authority >= MAX_COMMITTEE_SIZE { - return Err(io::Error::new( - error_kind, - format!( - "RBC-DAG frontier receipt carrier authority {} exceeds the maximum {}", - self.carrier_anchor.authority, - MAX_COMMITTEE_SIZE - 1 - ), - )); - } - Ok(()) - } - - pub(crate) fn to_bytes(&self) -> io::Result> { - self.validate(io::ErrorKind::InvalidInput)?; - - let mut bytes = Vec::with_capacity( - Self::FIXED_ENCODED_LEN - + self.committed_rounds.len() * std::mem::size_of::(), - ); - bytes.extend_from_slice(&Self::ENCODING_MAGIC); - bytes.extend_from_slice(&self.carrier_anchor.round.to_le_bytes()); - bytes.extend_from_slice(&self.carrier_anchor.authority.to_le_bytes()); - bytes.extend_from_slice(self.carrier_anchor.digest.as_array()); - bytes.extend_from_slice(&self.output_sequence.to_le_bytes()); - let watermark_count = u16::try_from(self.committed_rounds.len()).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - "RBC-DAG frontier receipt watermark count is not encodable", - ) - })?; - bytes.extend_from_slice(&watermark_count.to_le_bytes()); - for round in &self.committed_rounds { - bytes.extend_from_slice(&round.to_le_bytes()); - } - - Ok(bytes) - } - - pub(crate) fn from_bytes(bytes: &[u8]) -> io::Result { - if !(Self::FIXED_ENCODED_LEN..=Self::MAX_ENCODED_LEN).contains(&bytes.len()) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "stored RBC-DAG frontier receipt has an invalid encoded length", - )); - } - if bytes[..Self::ENCODING_MAGIC.len()] != Self::ENCODING_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "stored RBC-DAG frontier receipt has an unsupported encoding", - )); - } - - let mut cursor = Self::ENCODING_MAGIC.len(); - let take_u32 = |bytes: &[u8], cursor: &mut usize| { - let mut encoded = [0; std::mem::size_of::()]; - let end = *cursor + std::mem::size_of::(); - encoded.copy_from_slice(&bytes[*cursor..end]); - *cursor = end; - u32::from_le_bytes(encoded) - }; - let take_u16 = |bytes: &[u8], cursor: &mut usize| { - let mut encoded = [0; std::mem::size_of::()]; - let end = *cursor + std::mem::size_of::(); - encoded.copy_from_slice(&bytes[*cursor..end]); - *cursor = end; - u16::from_le_bytes(encoded) - }; - - let carrier_round = take_u32(bytes, &mut cursor); - let carrier_authority = take_u16(bytes, &mut cursor); - let mut carrier_digest = [0; BLOCK_DIGEST_SIZE]; - carrier_digest.copy_from_slice(&bytes[cursor..cursor + BLOCK_DIGEST_SIZE]); - cursor += BLOCK_DIGEST_SIZE; - let output_sequence = take_u32(bytes, &mut cursor); - let watermark_count = usize::from(take_u16(bytes, &mut cursor)); - if watermark_count == 0 || watermark_count > usize::from(MAX_COMMITTEE_SIZE) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "stored RBC-DAG frontier receipt has an invalid watermark count", - )); - } - let expected_len = - Self::FIXED_ENCODED_LEN + watermark_count * std::mem::size_of::(); - if bytes.len() != expected_len { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "stored RBC-DAG frontier receipt length does not match its watermark count", - )); - } - - let mut committed_rounds = Vec::with_capacity(watermark_count); - for _ in 0..watermark_count { - committed_rounds.push(take_u32(bytes, &mut cursor)); - } - let receipt = Self { - carrier_anchor: BlockReference { - round: carrier_round, - authority: carrier_authority, - digest: carrier_digest.into(), - }, - output_sequence, - committed_rounds, - }; - receipt.validate(io::ErrorKind::InvalidData)?; - Ok(receipt) - } -} - -/// Validate the exact atomic frontier write shape shared by every backend. -/// A nonempty batch contains precisely the current delta under its carrier -/// anchor; an empty batch is an explicit control-only marker. -pub(crate) fn validate_rbc_dag_frontier_commit_batch( - committed_sub_dags: &[CommitData], - receipt: &RbcDagFrontierReceipt, -) -> io::Result<()> { - receipt.validate(io::ErrorKind::InvalidInput)?; - if committed_sub_dags.len() > 1 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RBC-DAG frontier receipt batch must contain at most one application commit", - )); - } - if let Some(commit) = committed_sub_dags.first() { - if commit.sub_dag.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RBC-DAG frontier application commit must contain at least one application", - )); - } - if commit.leader != receipt.carrier_anchor { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RBC-DAG frontier application commit leader must equal the receipt carrier anchor", - )); - } - if commit.committed_rounds != receipt.committed_rounds { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RBC-DAG frontier application commit watermarks must equal the receipt watermarks", - )); - } - } - Ok(()) -} - /// Backend-agnostic storage interface for consensus blocks and commit data. /// /// Implementations must be thread-safe (`Send + Sync`). @@ -230,22 +34,8 @@ pub trait Store: Send + Sync + 'static { fn store_commits(&self, committed_sub_dags: Vec) -> io::Result<()>; - /// Atomically persist zero or more application commits and advance the - /// latest applied RBC-DAG frontier cursor. The carrier anchor is a - /// consensus-layer identity and is intentionally independent of any - /// application commit leader. A successful receipt read implies that all - /// commit data passed to this call is durable in the same batch. - fn store_commits_with_rbc_dag_receipt( - &self, - committed_sub_dags: Vec, - receipt: RbcDagFrontierReceipt, - ) -> io::Result<()>; - fn get_commit(&self, reference: &BlockReference) -> io::Result>; - /// Point-read the latest applied RBC-DAG frontier cursor in O(1). - fn read_latest_rbc_dag_frontier_receipt(&self) -> io::Result>; - // -- Component-level writes (pre-serialized) -- // Accept raw bincode bytes produced off the core thread by // `VerifiedBlock::preserialize()` or shard reconstructor workers. @@ -288,81 +78,3 @@ pub trait Store: Send + Sync + 'static { from_round: RoundNumber, ) -> io::Result>; } - -#[cfg(test)] -mod tests { - use super::{RbcDagFrontierReceipt, validate_rbc_dag_frontier_commit_batch}; - use crate::{dag_state::CommitData, types::BlockReference}; - - #[test] - fn rbc_dag_receipt_encoding_is_exact_and_bounded() { - let receipt = RbcDagFrontierReceipt { - carrier_anchor: BlockReference::new_test(3, 255), - output_sequence: 256, - committed_rounds: vec![250, 251, 252, 253], - }; - let encoded = receipt.to_bytes().unwrap(); - assert_eq!( - encoded.len(), - RbcDagFrontierReceipt::FIXED_ENCODED_LEN - + receipt.committed_rounds.len() * std::mem::size_of::() - ); - assert_eq!( - RbcDagFrontierReceipt::from_bytes(&encoded).unwrap(), - receipt - ); - - let mut trailing_byte = encoded.clone(); - trailing_byte.push(0); - assert_eq!( - RbcDagFrontierReceipt::from_bytes(&trailing_byte) - .unwrap_err() - .kind(), - std::io::ErrorKind::InvalidData - ); - - let mut mismatched_count = encoded; - let count_offset = RbcDagFrontierReceipt::FIXED_ENCODED_LEN - std::mem::size_of::(); - mismatched_count[count_offset..count_offset + std::mem::size_of::()] - .copy_from_slice(&5u16.to_le_bytes()); - assert_eq!( - RbcDagFrontierReceipt::from_bytes(&mismatched_count) - .unwrap_err() - .kind(), - std::io::ErrorKind::InvalidData - ); - - let zero_sequence = RbcDagFrontierReceipt { - carrier_anchor: BlockReference::new_test(0, 1), - output_sequence: 0, - committed_rounds: vec![0; 4], - }; - assert_eq!( - zero_sequence.to_bytes().unwrap_err().kind(), - std::io::ErrorKind::InvalidInput - ); - } - - #[test] - fn rbc_dag_frontier_commit_rejects_present_empty_application_data() { - let anchor = BlockReference::new_test(2, 7); - let receipt = RbcDagFrontierReceipt { - carrier_anchor: anchor, - output_sequence: 1, - committed_rounds: vec![0; 4], - }; - let empty_commit = CommitData { - leader: anchor, - sub_dag: Vec::new(), - committed_rounds: receipt.committed_rounds.clone(), - }; - - assert_eq!( - validate_rbc_dag_frontier_commit_batch(&[empty_commit], &receipt) - .unwrap_err() - .kind(), - std::io::ErrorKind::InvalidInput - ); - assert!(validate_rbc_dag_frontier_commit_batch(&[], &receipt).is_ok()); - } -} diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index ae1f9523..7f428984 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -17,7 +17,7 @@ use crate::{ bls_certificate_aggregator::{CertificateEvent, apply_certificate_events}, bls_service::BlsServiceMessage, consensus::{CommitMetastate, linearizer::CommittedSubDag}, - core::{Core, RbcDagFrontierApplyError, RbcDagFrontierApplyOutcome}, + core::Core, dag_state::{DagState, DataSource}, data::Data, metrics::Metrics, @@ -83,15 +83,6 @@ pub struct Syncer { starfish_rbc_service: Option, starfish_rbc_dag_shadow_service: Option, rbc_dag_frontier_authority: bool, - /// Production remains closed until the ordered service event bridge has - /// drained startup recovery and persisted every replayed frontier before - /// processing `Ready`. - rbc_dag_authority_ready: bool, - /// Exact locally produced application header waiting for durable carrier - /// assignment. Authoritative RBC-DAG mode permits only one outstanding - /// application so a fast legacy proposal clock cannot outrun the carrier - /// actor or turn a fixed queue size into a protocol parameter. - rbc_dag_pending_application: Option, } pub trait SyncerSignals: Send + Sync { @@ -106,9 +97,12 @@ pub trait CommitObserver: Send + Sync { committed_leaders: Vec<(Data, Option)>, ) -> Vec; - /// Observe an RBC-DAG frontier only after Core atomically persisted its - /// application commit (if any) and durable frontier receipt. - fn handle_rbc_dag_commit(&mut self, committed: &[CommittedSubDag]); + fn handle_rbc_dag_commit( + &mut self, + dag_state: &DagState, + anchor: BlockReference, + applications: &[BlockReference], + ) -> Vec; fn recover_committed( &mut self, @@ -121,7 +115,7 @@ pub trait CommitObserver: Send + Sync { impl Syncer { pub fn new( - core: Core, + mut core: Core, signals: S, commit_observer: C, metrics: Arc, @@ -131,6 +125,9 @@ impl Syncer { starfish_rbc_dag_shadow_service: Option, rbc_dag_frontier_authority: bool, ) -> Self { + if rbc_dag_frontier_authority { + core.enable_rbc_dag_application_production(); + } let committee_size = core.committee().len(); let own_stake = core .committee() @@ -153,8 +150,6 @@ impl Syncer { starfish_rbc_service, starfish_rbc_dag_shadow_service, rbc_dag_frontier_authority, - rbc_dag_authority_ready: !rbc_dag_frontier_authority, - rbc_dag_pending_application: None, } } @@ -166,32 +161,8 @@ impl Syncer { Vec, AHashSet, Vec, - ) { - if self.rbc_dag_frontier_authority { - tracing::warn!( - %source, - count = blocks.len(), - "Rejected generic block ingress while embedded RBC-DAG authority is active" - ); - return (Vec::new(), AHashSet::new(), Vec::new()); - } - self.add_blocks_inner(blocks, source) - } - - fn add_blocks_inner( - &mut self, - blocks: Vec<(Data, Option)>, - source: DataSource, - ) -> ( - Vec, - AHashSet, - Vec, ) { let previous_rounds = self.capture_rounds(); - let mut materialization_candidates = blocks - .iter() - .map(|(block, _)| *block.reference()) - .collect::>(); // todo: when block is updated we might return false here and it can make // committing longer let ( @@ -201,8 +172,6 @@ impl Syncer { used_additional_blocks, processed_blocks, ) = self.core.add_blocks(blocks, source); - materialization_candidates.extend(processed_blocks.iter().map(|block| *block.reference())); - self.notify_materialized_shadow_applications(materialization_candidates); if !processed_blocks.is_empty() { let block_refs: Vec<_> = processed_blocks.iter().map(|b| *b.reference()).collect(); self.send_sailfish_message(SailfishServiceMessage::ProcessBlocks(block_refs)); @@ -232,32 +201,10 @@ impl Syncer { &mut self, headers: Vec>, source: DataSource, - ) -> (AHashSet, Vec) { - if self.rbc_dag_frontier_authority { - tracing::warn!( - %source, - count = headers.len(), - "Rejected generic header ingress while embedded RBC-DAG authority is active" - ); - return (AHashSet::new(), Vec::new()); - } - self.add_headers_inner(headers, source) - } - - fn add_headers_inner( - &mut self, - headers: Vec>, - source: DataSource, ) -> (AHashSet, Vec) { let previous_rounds = self.capture_rounds(); - let mut materialization_candidates = headers - .iter() - .map(|header| *header.reference()) - .collect::>(); let (success, missing_parents, processed_refs, processed_blocks) = self.core.add_headers(headers, source); - materialization_candidates.extend(processed_refs.iter().copied()); - self.notify_materialized_shadow_applications(materialization_candidates); if !processed_blocks.is_empty() { // Send blocks to BLS service for verification of embedded BLS fields. self.send_bls_message(BlsServiceMessage::ProcessBlocks(processed_blocks.clone())); @@ -283,91 +230,11 @@ impl Syncer { items: Vec, source: DataSource, ) { - if self.rbc_dag_frontier_authority { - tracing::warn!( - %source, - count = items.len(), - "Rejected generic transaction-data ingress while embedded RBC-DAG authority is active" - ); - return; - } - self.add_transaction_data_inner(items, source); - } - - fn add_transaction_data_inner( - &mut self, - items: Vec, - source: DataSource, - ) { - let references = items - .iter() - .map(|item| item.block_reference) - .collect::>(); self.core.add_transaction_data(items, source); - self.notify_materialized_shadow_applications(references); self.maybe_update_proposal_wait(); self.try_new_block(BlockCreationReason::TransactionData); } - /// Materialize one application header whose authority was established by - /// the carrier actor. Keeping this as a separate core-thread command makes - /// the capability impossible to forge through `BlockBatch::source`. - pub(crate) fn add_authorized_rbc_dag_header( - &mut self, - header: RbcCanonicalHeader, - ) -> (AHashSet, Vec) { - assert!( - self.rbc_dag_frontier_authority, - "carrier-authorized header ingress requires embedded RBC-DAG authority" - ); - let mut block = header.to_authentication_free_block(); - block.preserialize(); - self.add_headers_inner( - vec![Data::new(block)], - DataSource::StarfishRbcDagAuthorizedHeader, - ) - } - - /// Attach payload data already verified against a carrier-authorized - /// canonical header. This cannot be invoked by a peer-controlled source - /// discriminator; only the typed core-thread command exposes it. - pub(crate) fn add_authorized_rbc_dag_payload(&mut self, item: ReconstructedTransactionData) { - assert!( - self.rbc_dag_frontier_authority, - "carrier-authorized payload ingress requires embedded RBC-DAG authority" - ); - self.add_transaction_data_inner(vec![item], DataSource::StarfishRbcDagAuthorizedPayload); - } - - /// Embedded RBC-DAG data availability is proven only by a concrete, - /// data-available DagState block. Payloads may arrive before their header - /// dependencies and remain buffered in Core; any later add path that - /// materializes the block retries this notification using the exact - /// processed references returned by Core. - fn notify_materialized_shadow_applications( - &self, - references: impl IntoIterator, - ) { - let Some(shadow) = self.starfish_rbc_dag_shadow_service.as_ref() else { - return; - }; - for reference in references { - if self.core.dag_state().get_storage_block(reference).is_none() - || !self.core.dag_state().is_data_available(&reference) - { - continue; - } - if let Err(error) = shadow.application_data_available(reference) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - tracing::warn!( - ?reference, - "Failed to record materialized RBC-DAG application availability: {error}" - ); - } - } - } - /// Called after Sailfish RBC certification events have been applied to /// DagState on the core thread. Retries block creation and sequencing /// when any clean vertex is new. @@ -413,10 +280,7 @@ impl Syncer { /// Sequence one exact deterministic carrier-frontier delta. In M7 this is /// the sole application-ordering authority; the legacy Starfish committer /// remains disabled in this mode. - pub fn apply_starfish_rbc_dag_frontier( - &mut self, - delta: CommittedFrontierDeltaV1, - ) -> Result { + pub fn apply_starfish_rbc_dag_frontier(&mut self, delta: CommittedFrontierDeltaV1) { assert!( self.rbc_dag_frontier_authority, "RBC-DAG frontier output requires the explicit authority mode" @@ -426,63 +290,13 @@ impl Syncer { .iter() .map(RbcCanonicalHeader::reference) .collect::>(); - match self.core.handle_rbc_dag_committed_delta( - delta.output_sequence, - delta.anchor, + let committed = self.commit_observer.handle_rbc_dag_commit( + self.core.dag_state(), + delta.anchor.carrier(), &applications, - )? { - RbcDagFrontierApplyOutcome::ExactReplay => Ok(false), - RbcDagFrontierApplyOutcome::Applied(committed) => { - self.commit_observer.handle_rbc_dag_commit(&committed); - self.try_new_block(BlockCreationReason::PostCommit); - Ok(true) - } - } - } - - /// Open the authoritative application-production gate only after the - /// service bridge observes `Ready`. Because the bridge awaits every prior - /// command, this is also a FIFO persistence barrier for recovery output. - pub(crate) fn activate_starfish_rbc_dag_authority(&mut self) { - assert!( - self.rbc_dag_frontier_authority, - "only embedded RBC-DAG authority mode has a startup barrier" - ); - if self.rbc_dag_authority_ready { - return; - } - self.rbc_dag_authority_ready = true; - self.core.enable_rbc_dag_application_production(); - let initial_round = self.core.next_block_round(); - self.force_new_block(initial_round); - } - - /// Acknowledge that the exact local application header is durably bound - /// into a carrier. This releases application production independently of - /// later RBC delivery/consensus commitment, preserving the carrier - /// pipeline while bounding producer lead to one header. - pub fn apply_starfish_rbc_dag_application_assigned(&mut self, reference: BlockReference) { - assert!( - self.rbc_dag_frontier_authority, - "RBC-DAG application assignment requires the explicit authority mode" ); - let Some(expected) = self.rbc_dag_pending_application else { - tracing::debug!( - ?reference, - "Ignoring stale RBC-DAG application-assignment acknowledgement" - ); - return; - }; - if expected != reference { - tracing::debug!( - ?expected, - ?reference, - "Ignoring RBC-DAG application-assignment acknowledgement for a different header" - ); - return; - } - self.rbc_dag_pending_application = None; - self.try_new_block(BlockCreationReason::CertificateEvent); + self.core.handle_rbc_dag_committed_delta(committed); + self.try_new_block(BlockCreationReason::PostCommit); } /// Store a Sailfish++ timeout certificate in DagState and retry block @@ -548,9 +362,6 @@ impl Syncer { /// round can lag the threshold clock) target the round they can /// actually enter. pub fn try_new_block_relaxed(&mut self, proposal_round: RoundNumber) -> bool { - if !self.rbc_dag_authority_ready || self.rbc_dag_pending_application.is_some() { - return false; - } if self.core.dag_state().proposal_round() != proposal_round { return false; } @@ -571,9 +382,6 @@ impl Syncer { } fn try_new_block(&mut self, reason: BlockCreationReason) -> bool { - if !self.rbc_dag_authority_ready || self.rbc_dag_pending_application.is_some() { - return false; - } self.maybe_update_proposal_wait(); if !self.core.committee().is_quorum(self.subscriber_stake) { return false; @@ -602,9 +410,6 @@ impl Syncer { } fn create_new_block(&mut self, reason: BlockCreationReason) -> bool { - if self.rbc_dag_pending_application.is_some() { - return false; - } tracing::debug!("Attempt to create new block in syncer after one trigger"); let previous_rounds = self.capture_rounds(); if let Some(ref block) = self.core.try_new_block(reason.as_str()) { @@ -619,18 +424,23 @@ impl Syncer { if self.core.dag_state().consensus_protocol.is_starfish_rbc() { let canonical = RbcCanonicalHeader::from_block_header(block.header()) .expect("locally built Starfish-RBC block must have canonical header content"); - if self.rbc_dag_frontier_authority { - self.rbc_dag_pending_application = Some(canonical.reference()); - let shadow = self - .starfish_rbc_dag_shadow_service - .as_ref() - .expect("embedded RBC-DAG authority must start its carrier service"); - if let Err(error) = shadow.local_application( - &canonical, - block.transaction_data().cloned().map(Arc::new), - ) { + let selected = self + .starfish_rbc_service + .as_ref() + .expect("Starfish-RBC protocol must start its RBC service") + .start_local_header_with_payload_blocking( + RbcLocalHeader::from_canonical(&canonical), + block.transaction_data().cloned(), + ) + .expect("local Starfish-RBC header must be accepted before dissemination"); + assert_eq!( + selected.reference(), + *block.reference(), + "RBC service selected a different local header reference" + ); + if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { + if let Err(error) = shadow.local_header(&canonical) { self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); self.metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["local", "dropped"]) @@ -638,35 +448,13 @@ impl Syncer { tracing::warn!( "Failed to enqueue RBC-DAG application carrier; the research run is invalid: {error}" ); - } else { - self.notify_materialized_shadow_applications([canonical.reference()]); - } - } else { - let selected = self - .starfish_rbc_service - .as_ref() - .expect("direct Starfish-RBC mode must start its RBC service") - .start_local_header_with_payload_blocking( - RbcLocalHeader::from_canonical(&canonical), - block.transaction_data().cloned(), - ) - .expect("local Starfish-RBC header must be accepted before dissemination"); - assert_eq!( - selected.reference(), - *block.reference(), - "RBC service selected a different local header reference" - ); - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.local_header(&canonical) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - self.metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["local", "dropped"]) - .inc(); - tracing::warn!("Failed to enqueue RBC-DAG mirror carrier: {error}"); - } else { - self.notify_materialized_shadow_applications([canonical.reference()]); - } + } else if let Err(error) = + shadow.application_data_available(canonical.reference()) + { + self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + tracing::warn!( + "Failed to record local RBC-DAG application availability: {error}" + ); } } } @@ -821,8 +609,6 @@ impl SyncerSignals for bool { #[cfg(test)] mod tests { - use std::time::Duration; - use prometheus::Registry; use tempfile::TempDir; @@ -831,21 +617,10 @@ mod tests { block_handler::BlockHandler, committee::Committee, config::{DisseminationMode, NodePrivateConfig, StorageBackend}, - crypto::{Signer, TransactionsCommitment, mac_keyrings_for_test}, + crypto::Signer, dag_state::{ConsensusProtocol, DagState}, - encoder::ShardEncoder, metrics::Metrics, - starfish_rbc_dag::{ - RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, - storage::ShadowWalSyncPolicyV1, - }, - starfish_rbc_dag_shadow::ShadowAuthorizerV1, - starfish_rbc_dag_shadow_service::{ - ShadowServiceEventV1, start_starfish_rbc_dag_autonomous_clock_service_v1, - }, - types::{ - BaseTransaction, BlockAuthenticationScheme, Encoder, Transaction, TransactionData, - }, + types::BaseTransaction, }; #[derive(Default)] @@ -871,7 +646,14 @@ mod tests { Vec::new() } - fn handle_rbc_dag_commit(&mut self, _committed: &[CommittedSubDag]) {} + fn handle_rbc_dag_commit( + &mut self, + _dag_state: &DagState, + _anchor: BlockReference, + _applications: &[BlockReference], + ) -> Vec { + Vec::new() + } fn recover_committed( &mut self, @@ -1114,319 +896,6 @@ mod tests { Data::new(block) } - fn make_starfish_rbc_round_1_application( - committee: &Committee, - authority: AuthorityIndex, - receiver: AuthorityIndex, - ) -> ( - Data, - RbcCanonicalHeader, - ReconstructedTransactionData, - ) { - let payload_byte = u8::try_from(authority).unwrap(); - let transactions = vec![BaseTransaction::Share(Transaction::new(vec![ - payload_byte; - 64 - ]))]; - let mut encoder = Encoder::new(2, 4, 2).unwrap(); - let encoded = encoder.encode_transactions( - &transactions, - committee.info_length(), - committee.len() - committee.info_length(), - ); - let mut block = VerifiedBlock::new_starfish_rbc( - authority, - 1, - committee - .authorities() - .map(|authority| BlockReference::new_test(authority, 0)) - .collect(), - Vec::new(), - u64::from(authority) + 1, - transactions.clone(), - Some(encoded.clone()), - ); - let canonical = RbcCanonicalHeader::from_block_header(block.header()).unwrap(); - block.preserialize(); - - let mut transaction_data = TransactionData::new(transactions); - transaction_data.preserialize(); - let (commitment, proof) = - TransactionsCommitment::new_from_encoded_transactions(&encoded, receiver as usize); - assert_eq!(commitment, canonical.transactions_commitment()); - let mut shard_data = ProvableShard::new( - encoded[receiver as usize].clone(), - receiver as usize, - proof, - commitment, - ); - shard_data.preserialize(); - let payload = ReconstructedTransactionData { - block_reference: canonical.reference(), - transaction_data, - shard_data, - }; - (Data::new(block), canonical, payload) - } - - fn make_reconstructed_payload( - full_block: &Data, - committee: &Committee, - receiver: AuthorityIndex, - ) -> ReconstructedTransactionData { - let transactions = full_block - .transaction_data() - .expect("test application must carry transaction data") - .transactions() - .clone(); - let mut encoder = Encoder::new(2, 4, 2).unwrap(); - let encoded = encoder.encode_transactions( - &transactions, - committee.info_length(), - committee.len() - committee.info_length(), - ); - let mut transaction_data = TransactionData::new(transactions); - transaction_data.preserialize(); - let (commitment, proof) = - TransactionsCommitment::new_from_encoded_transactions(&encoded, receiver as usize); - let mut shard_data = ProvableShard::new( - encoded[receiver as usize].clone(), - receiver as usize, - proof, - commitment, - ); - shard_data.preserialize(); - ReconstructedTransactionData { - block_reference: *full_block.reference(), - transaction_data, - shard_data, - } - } - - async fn wait_for_shadow_ready(events: &mut mpsc::Receiver) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - match events.recv().await { - Some(ShadowServiceEventV1::Ready { autonomous_clock }) => { - assert!(autonomous_clock); - break; - } - Some(_) => {} - None => panic!("autonomous shadow service stopped before Ready"), - } - } - }) - .await - .expect("autonomous shadow service did not become ready"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn rbc_dag_gate_and_typed_ingress_are_enforced() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let recovered = DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ); - let private_config = NodePrivateConfig::new_for_tests(authority); - let (core, _) = Core::open( - TestBlockHandler, - authority, - committee.clone(), - private_config, - metrics.clone(), - recovered, - None, - ); - - let shadow_directory = TempDir::new().unwrap(); - let keyrings = mac_keyrings_for_test(committee.len()); - let committee_context = RbcDagCommitteeContextV1::new(committee.clone()).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0x53; 32]).unwrap(), - &committee_context, - BlockAuthenticationScheme::MacVector, - ); - let (shadow_service, mut shadow_events, shadow_task) = - start_starfish_rbc_dag_autonomous_clock_service_v1( - shadow_directory.path().join("standalone.wal"), - committee_context, - authority, - context, - ShadowAuthorizerV1::MacVector(keyrings[authority as usize].clone()), - Vec::new(), - Duration::from_secs(3_600), - ShadowWalSyncPolicyV1::EveryBatch, - ) - .unwrap(); - wait_for_shadow_ready(&mut shadow_events).await; - // This unit drives the Syncer boundary directly instead of running the - // production event bridge. Keep draining the actor's bounded event - // channel so local carrier fanout cannot block the actor (and hence a - // later graceful shutdown) behind events this test intentionally does - // not consume. - let shadow_event_drain = - tokio::spawn(async move { while shadow_events.recv().await.is_some() {} }); - - let mut syncer = Syncer::new( - core, - TestSignals::default(), - NoopCommitObserver, - metrics, - None, - None, - None, - Some(shadow_service.clone()), - true, - ); - syncer.connected_authorities.extend([1, 2, 3]); - syncer.subscribed_by_authorities.extend([1, 2, 3]); - syncer.recompute_subscriber_stake(); - - // Production is intentionally closed until the ordered service bridge - // consumes `Ready`. Exercise that authority barrier before asserting - // the one-outstanding application gate. - syncer.activate_starfish_rbc_dag_authority(); - let first = syncer - .rbc_dag_pending_application - .expect("round-one application must wait for carrier assignment"); - assert_eq!(first.round, 1); - assert_eq!(syncer.core.last_proposed(), 1); - assert_eq!(syncer.signals.new_block_ready_count, 1); - assert!(!shadow_task.is_finished()); - - // Peer-controlled generic ingress cannot materialize authentication- - // free applications in standalone mode. - let (full_one, canonical_one, payload_one) = - make_starfish_rbc_round_1_application(&committee, 1, authority); - let reference_one = canonical_one.reference(); - let mut header_one = canonical_one.to_authentication_free_block(); - header_one.preserialize(); - let generic_headers = syncer.add_headers( - vec![Data::new(header_one)], - DataSource::BlockBundleStreamingHeader, - ); - assert!(generic_headers.0.is_empty()); - assert!(generic_headers.1.is_empty()); - assert!( - syncer - .core - .dag_state() - .get_storage_block(reference_one) - .is_none() - ); - - let (full_two, canonical_two, _) = - make_starfish_rbc_round_1_application(&committee, 2, authority); - let reference_two = canonical_two.reference(); - let generic_blocks = - syncer.add_blocks(vec![(full_two, None)], DataSource::BlockBundleStreaming); - assert!(generic_blocks.0.is_empty()); - assert!(generic_blocks.1.is_empty()); - assert!(generic_blocks.2.is_empty()); - assert!( - syncer - .core - .dag_state() - .get_storage_block(reference_two) - .is_none() - ); - - // The actor-only typed header command admits the exact canonical - // application, while the generic payload command remains closed. - let (missing_one, _) = syncer.add_authorized_rbc_dag_header(canonical_one); - assert!(missing_one.is_empty()); - assert!( - syncer - .core - .dag_state() - .get_storage_block(reference_one) - .is_some() - ); - assert!(!syncer.core.dag_state().is_data_available(&reference_one)); - - syncer.add_transaction_data( - vec![payload_one], - DataSource::StarfishRbcDagAuthorizedPayload, - ); - assert!(!syncer.core.dag_state().is_data_available(&reference_one)); - - syncer.add_authorized_rbc_dag_payload(make_reconstructed_payload( - &full_one, &committee, authority, - )); - assert!(syncer.core.dag_state().is_data_available(&reference_one)); - - let (missing_two, _) = syncer.add_authorized_rbc_dag_header(canonical_two); - assert!(missing_two.is_empty()); - assert!( - syncer - .core - .dag_state() - .get_storage_block(reference_two) - .is_some() - ); - - // Even after a typed header quorum advances the application clock, - // all creation triggers remain closed while the first header is pending. - assert_eq!(syncer.core.dag_state().threshold_clock_round(), 2); - assert!(!syncer.try_new_block(BlockCreationReason::NewHeaders)); - assert_eq!(syncer.core.last_proposed(), 1); - assert_eq!(syncer.rbc_dag_pending_application, Some(first)); - - let wrong = BlockReference::new_test(3, first.round); - syncer.apply_starfish_rbc_dag_application_assigned(wrong); - assert_eq!(syncer.rbc_dag_pending_application, Some(first)); - assert_eq!(syncer.core.last_proposed(), 1); - - // The exact acknowledgement releases one proposal attempt. That - // attempt creates round two and immediately closes the gate around - // the new outstanding header; it cannot create more than one block. - syncer.apply_starfish_rbc_dag_application_assigned(first); - let second = syncer - .rbc_dag_pending_application - .expect("round-two application must become the sole outstanding header"); - assert_eq!(second.round, 2); - assert_ne!(second, first); - assert_eq!(syncer.core.last_proposed(), 2); - assert_eq!(syncer.signals.new_block_ready_count, 2); - assert!(!syncer.try_new_block(BlockCreationReason::CertificateEvent)); - assert_eq!(syncer.core.last_proposed(), 2); - - // A duplicate acknowledgement for the old header is stale and must - // neither release nor replace the current gate. - syncer.apply_starfish_rbc_dag_application_assigned(first); - assert_eq!(syncer.rbc_dag_pending_application, Some(second)); - assert_eq!(syncer.core.last_proposed(), 2); - - // With no round-three quorum, the exact second acknowledgement simply - // clears the gate. A later duplicate is harmless and leaves it clear. - syncer.apply_starfish_rbc_dag_application_assigned(second); - assert_eq!(syncer.rbc_dag_pending_application, None); - assert_eq!(syncer.core.last_proposed(), 2); - syncer.apply_starfish_rbc_dag_application_assigned(second); - assert_eq!(syncer.rbc_dag_pending_application, None); - assert_eq!(syncer.signals.new_block_ready_count, 2); - - shadow_service.shutdown().await.unwrap(); - shadow_task.await.unwrap(); - shadow_event_drain.await.unwrap(); - } - #[test] fn normal_block_creation_uses_next_missing_round() { let mut syncer = open_test_syncer_with_future_rounds(); diff --git a/crates/starfish-core/src/tidehunter_store.rs b/crates/starfish-core/src/tidehunter_store.rs index e9b6c35d..1339b669 100644 --- a/crates/starfish-core/src/tidehunter_store.rs +++ b/crates/starfish-core/src/tidehunter_store.rs @@ -15,7 +15,7 @@ use tidehunter::{ use crate::{ dag_state::CommitData, data::Data, - store::{RbcDagFrontierReceipt, Store, validate_rbc_dag_frontier_commit_batch}, + store::Store, types::{ BlockHeader, BlockReference, ProvableShard, RoundNumber, TransactionData, VerifiedBlock, }, @@ -31,7 +31,6 @@ const PREFIX_LEN: usize = 3; /// Number of mutexes per key space for concurrency control. Must be power of 2. const MUTEXES: usize = 64; -const LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY: [u8; KEY_SIZE] = [0; KEY_SIZE]; pub struct TideHunterStore { db: Arc, @@ -41,7 +40,6 @@ pub struct TideHunterStore { ks_shard_data: KeySpace, ks_commits: KeySpace, ks_dual_dag_clean: KeySpace, - ks_rbc_dag_frontier_receipt: KeySpace, } impl TideHunterStore { @@ -85,7 +83,6 @@ impl TideHunterStore { let ks_shard_data = Self::add_ks(&mut builder, "shard_data"); let ks_commits = Self::add_ks(&mut builder, "commits"); let ks_dual_dag_clean = Self::add_ks(&mut builder, "sailfish_certified"); - let ks_rbc_dag_frontier_receipt = Self::add_ks(&mut builder, "rbc_dag_frontier_receipt"); let key_shape = builder.build(); let config = Arc::new(Config { @@ -108,7 +105,6 @@ impl TideHunterStore { ks_shard_data, ks_commits, ks_dual_dag_clean, - ks_rbc_dag_frontier_receipt, }) } @@ -251,36 +247,6 @@ impl Store for TideHunterStore { .map_err(|e| io::Error::other(format!("TideHunter commit batch: {e:?}"))) } - fn store_commits_with_rbc_dag_receipt( - &self, - committed_sub_dags: Vec, - receipt: RbcDagFrontierReceipt, - ) -> io::Result<()> { - validate_rbc_dag_frontier_commit_batch(&committed_sub_dags, &receipt)?; - let receipt_bytes = receipt.to_bytes()?; - - let mut batch = self.db.write_batch(); - if committed_sub_dags.is_empty() { - batch.delete( - self.ks_commits, - Self::encode_key(&receipt.carrier_anchor).to_vec(), - ); - } else { - let commit_data = &committed_sub_dags[0]; - let key = Self::encode_key(&commit_data.leader); - let value = bincode::serialize(&commit_data).map_err(io::Error::other)?; - batch.write(self.ks_commits, key.to_vec(), value); - } - batch.write( - self.ks_rbc_dag_frontier_receipt, - LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY.to_vec(), - receipt_bytes, - ); - batch.commit().map_err(|e| { - io::Error::other(format!("TideHunter RBC-DAG commit/receipt batch: {e:?}")) - }) - } - fn get_commit(&self, reference: &BlockReference) -> io::Result> { let key = Self::encode_key(reference); match self @@ -297,20 +263,6 @@ impl Store for TideHunterStore { } } - fn read_latest_rbc_dag_frontier_receipt(&self) -> io::Result> { - match self - .db - .get( - self.ks_rbc_dag_frontier_receipt, - &LATEST_RBC_DAG_FRONTIER_RECEIPT_KEY, - ) - .map_err(|e| io::Error::other(format!("TideHunter get receipt: {e:?}")))? - { - Some(bytes) => RbcDagFrontierReceipt::from_bytes(&bytes).map(Some), - None => Ok(None), - } - } - fn store_header_bytes(&self, reference: &BlockReference, bytes: &[u8]) -> io::Result<()> { let key = Self::encode_key(reference); self.db @@ -452,32 +404,8 @@ impl Store for TideHunterStore { #[cfg(test)] mod tests { - use tempfile::TempDir; - use super::TideHunterStore; - use crate::{ - dag_state::CommitData, - store::{RbcDagFrontierReceipt, Store}, - types::{BlockReference, MAX_COMMITTEE_SIZE}, - }; - - fn commit(leader: BlockReference, committed_rounds: Vec) -> CommitData { - CommitData { - leader, - sub_dag: vec![BlockReference::new_test(1, leader.round)], - committed_rounds, - } - } - - fn assert_commit(store: &impl Store, expected: &CommitData) { - let actual = store - .get_commit(&expected.leader) - .expect("commit read should succeed") - .expect("commit should exist"); - assert_eq!(actual.leader, expected.leader); - assert_eq!(actual.sub_dag, expected.sub_dag); - assert_eq!(actual.committed_rounds, expected.committed_rounds); - } + use crate::types::BlockReference; #[test] fn encode_key_preserves_u16_authority() { @@ -501,129 +429,4 @@ mod tests { assert_eq!(&low_key[4..6], &255u16.to_be_bytes()); assert_eq!(&high_key[4..6], &256u16.to_be_bytes()); } - - #[test] - fn rbc_dag_receipt_and_commits_are_atomic_and_latest_is_a_point_value() { - let temp_dir = TempDir::new().unwrap(); - let store = TideHunterStore::open(temp_dir.path()).unwrap(); - - let legacy_leader = BlockReference::new_test(2, 253); - let legacy_commit = commit(legacy_leader, vec![253; 4]); - store.store_commits(vec![legacy_commit.clone()]).unwrap(); - assert_commit(&store, &legacy_commit); - assert!( - store - .read_latest_rbc_dag_frontier_receipt() - .unwrap() - .is_none() - ); - - // A control-only frontier has no new application commits, but its - // durable cursor must still advance. - let first_anchor = BlockReference::new_test(7, 255); - let first_receipt = RbcDagFrontierReceipt { - carrier_anchor: first_anchor, - output_sequence: 255, - committed_rounds: vec![250, 251, 252, 253], - }; - let stale_first_commit = commit(first_anchor, first_receipt.committed_rounds.clone()); - store.store_commits(vec![stale_first_commit]).unwrap(); - assert!(store.get_commit(&first_anchor).unwrap().is_some()); - store - .store_commits_with_rbc_dag_receipt(Vec::new(), first_receipt.clone()) - .unwrap(); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(first_receipt) - ); - assert!(store.get_commit(&first_anchor).unwrap().is_none()); - - // The exact application commit is stored under the consensus carrier - // anchor so Core can reconstruct the compact receipt's application - // references after restart. - let second_anchor = BlockReference::new_test(7, 256); - let application_commit = commit(second_anchor, vec![255, 256, 255, 256]); - let second_receipt = RbcDagFrontierReceipt { - carrier_anchor: second_anchor, - output_sequence: 256, - committed_rounds: vec![255, 256, 255, 256], - }; - store - .store_commits_with_rbc_dag_receipt( - vec![application_commit.clone()], - second_receipt.clone(), - ) - .unwrap(); - assert_commit(&store, &application_commit); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(second_receipt.clone()) - ); - - // Mismatched/multiple application commits are rejected before either - // commit data or the latest receipt can change. - let mismatched = commit(BlockReference::new_test(2, 254), vec![255, 256, 255, 256]); - let mismatched_watermarks = commit(second_anchor, vec![1; 4]); - for invalid in [ - vec![mismatched], - vec![mismatched_watermarks], - vec![application_commit.clone(), application_commit.clone()], - ] { - let error = store - .store_commits_with_rbc_dag_receipt(invalid, second_receipt.clone()) - .unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(second_receipt.clone()) - ); - } - - // Reusing the exact anchor for a control-only marker atomically - // removes stale application CommitData, preserving absence semantics. - let control_receipt = RbcDagFrontierReceipt { - carrier_anchor: second_anchor, - output_sequence: 257, - committed_rounds: second_receipt.committed_rounds.clone(), - }; - store - .store_commits_with_rbc_dag_receipt(Vec::new(), control_receipt.clone()) - .unwrap(); - assert!(store.get_commit(&second_anchor).unwrap().is_none()); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(control_receipt.clone()) - ); - - // Receipt validation happens before the batch is submitted, so an - // invalid vector cannot partially write its application commit or - // replace the last valid cursor. - let rejected_leader = BlockReference::new_test(3, 257); - let rejected = commit(rejected_leader, vec![257; 4]); - for committed_rounds in [Vec::new(), vec![0; usize::from(MAX_COMMITTEE_SIZE) + 1]] { - let invalid = RbcDagFrontierReceipt { - carrier_anchor: BlockReference::new_test(7, 257), - output_sequence: 258, - committed_rounds, - }; - let error = store - .store_commits_with_rbc_dag_receipt(vec![rejected.clone()], invalid) - .unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - } - assert!(store.get_commit(&rejected_leader).unwrap().is_none()); - assert_eq!( - store.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(control_receipt.clone()) - ); - - drop(store); - let reopened = TideHunterStore::open(temp_dir.path()).unwrap(); - assert_commit(&reopened, &legacy_commit); - assert!(reopened.get_commit(&second_anchor).unwrap().is_none()); - assert_eq!( - reopened.read_latest_rbc_dag_frontier_receipt().unwrap(), - Some(control_receipt) - ); - } } diff --git a/crates/starfish-core/src/transactions_generator.rs b/crates/starfish-core/src/transactions_generator.rs index 285c316e..0ca474f9 100644 --- a/crates/starfish-core/src/transactions_generator.rs +++ b/crates/starfish-core/src/transactions_generator.rs @@ -5,19 +5,16 @@ use std::{ cmp::min, sync::{Arc, atomic::Ordering}, - time::Duration, + time::{Duration, Instant}, }; use rand::{Rng, RngCore, SeedableRng, rngs::StdRng}; -use tokio::{ - sync::{mpsc, watch}, - time::{Instant, MissedTickBehavior, interval_at, sleep_until}, -}; +use tokio::sync::mpsc; use crate::{ config::{NodePublicConfig, Parameters, TransactionMode}, crypto::AsBytes, - metrics::{BenchmarkGeneratorState, BenchmarkTransactionWindow, Metrics}, + metrics::Metrics, runtime::{self, timestamp_utc}, types::{AuthorityIndex, Transaction}, }; @@ -28,14 +25,6 @@ pub struct TransactionGenerator { parameters: Parameters, node_public_config: NodePublicConfig, metrics: Arc, - start_gate: Option>>, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum SubmissionOutcome { - Submitted, - Cutoff, - Closed, } impl TransactionGenerator { @@ -50,73 +39,14 @@ impl TransactionGenerator { transactions } - async fn submit_before_end( - &self, - block: Vec, - generation_end: Option, - ) -> SubmissionOutcome { - let Some(end) = generation_end else { - return if self.sender.send(block).await.is_ok() { - SubmissionOutcome::Submitted - } else { - SubmissionOutcome::Closed - }; - }; - if Instant::now() >= end { - return SubmissionOutcome::Cutoff; - } - // Deadline first makes t == end exclusive even when channel capacity - // and the cutoff become ready in the same scheduler turn. A pending - // send is cancelled without publishing its block, so it cannot become - // post-window offered work or drain debt. - tokio::select! { - biased; - _ = sleep_until(end) => SubmissionOutcome::Cutoff, - result = self.sender.send(block) => { - if result.is_ok() { - SubmissionOutcome::Submitted - } else { - SubmissionOutcome::Closed - } - } - } - } - - #[cfg(test)] pub fn start( sender: mpsc::Sender>, seed: AuthorityIndex, parameters: Parameters, node_public_config: NodePublicConfig, metrics: Arc, - ) { - Self::start_with_gate(sender, seed, parameters, node_public_config, metrics, None); - } - - pub fn start_with_gate( - sender: mpsc::Sender>, - seed: AuthorityIndex, - parameters: Parameters, - node_public_config: NodePublicConfig, - metrics: Arc, - start_gate: Option>>, ) { assert!(parameters.transaction_size > 8 + 8); // 8 bytes timestamp + 8 bytes random - // Publish the finite-window warmup state synchronously. The spawned - // task may not be polled until after the benchmark harness inspects - // readiness, so setting this only inside `run` creates a false active - // window and can baseline before the topology is complete. - if parameters.benchmark_duration.is_some() { - metrics.metrics_active.store(false, Ordering::Relaxed); - metrics - .transaction_metrics_active - .store(false, Ordering::Relaxed); - } - if start_gate.is_some() { - metrics - .benchmark_generator_state - .store(BenchmarkGeneratorState::Waiting as u8, Ordering::Release); - } runtime::Handle::current().spawn( Self { sender, @@ -124,7 +54,6 @@ impl TransactionGenerator { parameters, node_public_config, metrics, - start_gate, } .run(), ); @@ -133,13 +62,9 @@ impl TransactionGenerator { pub async fn run(mut self) { let load = self.parameters.load; let max_transactions_per_block_interval = load.div_ceil(Self::BATCHES_IN_SECOND); - let coordinated_start = self.start_gate.is_some(); // Add a small extra delay proportional to committee size to give - // connections time to come up in normal production/ungated runs. - // Calibrated so n=100 lands at ~15s total when initial_delay is the - // 10s default. A coordinated benchmark gate is released only after - // topology and clock activation, so it deliberately skips this - // second warmup and starts the shared finite window immediately. + // connections time to come up. Calibrated so n=100 lands at ~15s + // total when initial_delay is the 10s default. let initial_delay_plus_extra_delay = self.parameters.initial_delay + Duration::from_millis( (self.node_public_config.identifiers.len() as f64 / 100.0 * 5000.0) as u64, @@ -147,30 +72,24 @@ impl TransactionGenerator { let benchmark_duration = self.parameters.benchmark_duration; // When the orchestrator sets a finite benchmark window, gate metrics - // off while awaiting either the coordinated release or the normal - // warmup so `benchmark_duration` counts only active submissions. - let finite_window_description = match benchmark_duration { - Some(d) => format!(", stopping after {} sec of generation", d.as_secs()), - None => String::new(), - }; - if coordinated_start { - tracing::info!( - "Starting tx generator behind the coordinated release gate; \ - targeting {load} tx/s immediately after release \ - (up to {max_transactions_per_block_interval} transactions every {} ms){}", - Self::TARGET_BLOCK_INTERVAL.as_millis(), - finite_window_description, - ); - } else { - tracing::info!( - "Starting tx generator. After {} sec, \ - targeting {load} tx/s \ - (up to {max_transactions_per_block_interval} transactions every {} ms){}", - initial_delay_plus_extra_delay.as_secs(), - Self::TARGET_BLOCK_INTERVAL.as_millis(), - finite_window_description, - ); + // off during the warmup so the validator's `benchmark_duration` + // counter only ticks inside the active submission window. Without + // this, the warmup seconds inflate the TPS denominator. + if benchmark_duration.is_some() { + self.metrics.metrics_active.store(false, Ordering::Relaxed); } + + tracing::info!( + "Starting tx generator. After {} sec, \ + targeting {load} tx/s \ + (up to {max_transactions_per_block_interval} transactions every {} ms){}", + initial_delay_plus_extra_delay.as_secs(), + Self::TARGET_BLOCK_INTERVAL.as_millis(), + match benchmark_duration { + Some(d) => format!(", stopping after {} sec of generation", d.as_secs()), + None => String::new(), + }, + ); let max_block_size = self.node_public_config.parameters.max_block_size; let target_block_size = min(max_block_size, max_transactions_per_block_interval); @@ -182,69 +101,46 @@ impl TransactionGenerator { }); let mut counter: u64 = 0; + let mut tx_to_report = 0; let mut random: u64 = self.rng.gen(); let mut load_carry = 0; // Pre-allocated payload buffer reused in AllZero mode. let zeros = vec![0u8; tx_size - 8 - 8]; - let coordinated_window = if let Some(mut start_gate) = self.start_gate.take() { - match start_gate.wait_for(Option::is_some).await { - Ok(window) => *window, - Err(_) => { - self.fail_coordinated_window(); - return; - } - } - } else { - None - }; - - if !coordinated_start { - runtime::sleep(initial_delay_plus_extra_delay).await; - } - - let generation_start = coordinated_window - .map(|window| window.start) - .unwrap_or_else(Instant::now); - let generation_end = coordinated_window.map(|window| window.end).or_else(|| { - benchmark_duration.and_then(|duration| generation_start.checked_add(duration)) - }); - if Instant::now() < generation_start { - sleep_until(generation_start).await; - } - - // Anchor every coordinated generator to the same absolute tick grid. - // MissedTickBehavior::Skip prevents a delayed task from bursting old - // batches, and comparing the scheduled tick strictly with `end` - // excludes the historical extra batch at t == duration. - let mut interval = interval_at(generation_start, Self::TARGET_BLOCK_INTERVAL); - interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut interval = runtime::TimeInterval::new(Self::TARGET_BLOCK_INTERVAL); + runtime::sleep(initial_delay_plus_extra_delay).await; + let generation_start = Instant::now(); // Open the active metrics window: anchor `benchmark_duration`'s // clock to this instant so the TPS denominator only counts seconds // during which transactions are actually being submitted. - let active_start_micros = generation_start - .saturating_duration_since(self.metrics.validator_start) + let active_start_micros = self + .metrics + .validator_start + .elapsed() .as_micros() .min(u64::MAX as u128) as u64; self.metrics .active_start_micros .store(active_start_micros, Ordering::Relaxed); self.metrics.metrics_active.store(true, Ordering::Relaxed); - self.metrics - .transaction_metrics_active - .store(true, Ordering::Relaxed); - if coordinated_start { - self.metrics - .benchmark_generator_state - .store(BenchmarkGeneratorState::Active as u8, Ordering::Release); - } - 'generation: loop { - let scheduled_tick = interval.tick().await; - if generation_end.is_some_and(|end| scheduled_tick >= end) { - break; + loop { + if let Some(limit) = benchmark_duration { + if generation_start.elapsed() >= limit { + tracing::info!( + "Tx generator reached benchmark duration ({} sec); \ + stopping submissions and closing the metrics window.", + limit.as_secs(), + ); + // Close the active metrics window so commits arriving + // during the wind-down don't pollute cumulative + // quantiles or skew the TPS denominator. + self.metrics.metrics_active.store(false, Ordering::Relaxed); + break; + } } + interval.tick().await; let timestamp = (timestamp_utc().as_millis() as u64).to_le_bytes(); let transactions_per_block_interval = Self::transactions_for_interval(load, &mut load_carry); @@ -271,73 +167,28 @@ impl TransactionGenerator { block.push(Transaction::new(transaction)); block_size += tx_size; counter += 1; + tx_to_report += 1; if block_size >= max_block_size { - let submitted = block.len() as u64; - match self.submit_before_end(block.clone(), generation_end).await { - SubmissionOutcome::Submitted => {} - SubmissionOutcome::Cutoff => break 'generation, - SubmissionOutcome::Closed => { - self.fail_coordinated_window(); - return; - } + if self.sender.send(block.clone()).await.is_err() { + return; } - self.metrics.submitted_transactions.inc_by(submitted); - self.metrics - .submitted_transactions_bytes - .inc_by(submitted.saturating_mul(tx_size as u64)); block.clear(); block_size = 0; } } tracing::debug!("Generator send {} transactions", block.len()); - if !block.is_empty() { - let submitted = block.len() as u64; - match self.submit_before_end(block, generation_end).await { - SubmissionOutcome::Submitted => {} - SubmissionOutcome::Cutoff => break 'generation, - SubmissionOutcome::Closed => { - self.fail_coordinated_window(); - return; - } - } - self.metrics.submitted_transactions.inc_by(submitted); - self.metrics - .submitted_transactions_bytes - .inc_by(submitted.saturating_mul(tx_size as u64)); + if !block.is_empty() && self.sender.send(block).await.is_err() { + return; } - } - - self.metrics.metrics_active.store(false, Ordering::Release); - if coordinated_start { - // The local benchmark keeps transaction observation open during - // its bounded drain and closes it after every offered transaction - // is observed (or reports an explicit incomplete drain). - self.metrics - .benchmark_generator_state - .store(BenchmarkGeneratorState::Finished as u8, Ordering::Release); - } else { - self.metrics - .transaction_metrics_active - .store(false, Ordering::Release); - } - } - fn fail_coordinated_window(&self) { - self.metrics.metrics_active.store(false, Ordering::Release); - self.metrics - .transaction_metrics_active - .store(false, Ordering::Release); - if self.start_gate.is_some() - || BenchmarkGeneratorState::from_u8( + if counter.is_multiple_of(10_000) { self.metrics - .benchmark_generator_state - .load(Ordering::Acquire), - ) != BenchmarkGeneratorState::Disabled - { - self.metrics - .benchmark_generator_state - .store(BenchmarkGeneratorState::Failed as u8, Ordering::Release); + .submitted_transactions_bytes + .inc_by(tx_to_report * tx_size as u64); + self.metrics.submitted_transactions.inc_by(tx_to_report); + tx_to_report = 0 + } } } @@ -351,23 +202,7 @@ impl TransactionGenerator { #[cfg(test)] mod tests { - use std::{ - net::{IpAddr, Ipv4Addr}, - sync::atomic::Ordering, - time::Duration, - }; - - use prometheus::Registry; - use tokio::{ - sync::{mpsc, watch}, - time::{Instant, timeout}, - }; - use super::TransactionGenerator; - use crate::{ - config::{NodePublicConfig, Parameters}, - metrics::{BenchmarkGeneratorState, BenchmarkTransactionWindow, Metrics}, - }; #[test] fn transactions_for_interval_matches_target_rate() { @@ -381,175 +216,4 @@ mod tests { assert_eq!(carry, 0, "load={load}"); } } - - #[tokio::test] - async fn ungated_finite_generator_keeps_metrics_closed_during_warmup() { - let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); - metrics.metrics_active.store(true, Ordering::Relaxed); - let (sender, mut receiver) = mpsc::channel(1); - let mut parameters = Parameters::almost_default(1); - parameters.benchmark_duration = Some(Duration::from_secs(1)); - parameters.initial_delay = Duration::from_secs(60); - let public_config = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); - - TransactionGenerator::start(sender, 0, parameters, public_config, metrics.clone()); - - assert!(!metrics.metrics_active.load(Ordering::Relaxed)); - assert!( - timeout(Duration::from_millis(100), receiver.recv()) - .await - .is_err(), - "ungated generator skipped its configured warmup" - ); - assert!(!metrics.metrics_active.load(Ordering::Relaxed)); - } - - #[tokio::test] - async fn coordinated_generator_starts_immediately_after_release() { - let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); - let (sender, mut receiver) = mpsc::channel(1); - let mut parameters = Parameters::almost_default(20); - parameters.benchmark_duration = Some(Duration::from_secs(1)); - parameters.initial_delay = Duration::from_secs(60); - let public_config = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); - let (release, gate) = watch::channel(None); - - TransactionGenerator::start_with_gate( - sender, - 0, - parameters, - public_config, - metrics.clone(), - Some(gate), - ); - assert!( - timeout(Duration::from_millis(100), receiver.recv()) - .await - .is_err(), - "generator submitted before the coordinated release" - ); - assert!(!metrics.metrics_active.load(Ordering::Relaxed)); - - let start = Instant::now() + Duration::from_millis(10); - release.send_replace(BenchmarkTransactionWindow::new( - start, - start + Duration::from_secs(1), - )); - timeout(Duration::from_millis(500), receiver.recv()) - .await - .expect("released generator waited for the configured initial delay") - .expect("released generator channel closed"); - assert!(metrics.metrics_active.load(Ordering::Relaxed)); - - assert!( - timeout(Duration::from_millis(20), receiver.recv()) - .await - .is_err(), - "released generator emitted a stale-interval catch-up burst" - ); - timeout(Duration::from_millis(250), receiver.recv()) - .await - .expect("released generator did not establish a fresh cadence") - .expect("released generator channel closed"); - } - - #[tokio::test] - async fn coordinated_window_counts_every_send_and_excludes_end_tick() { - let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); - let (sender, mut receiver) = mpsc::channel(16); - let mut parameters = Parameters::almost_default(20); - parameters.benchmark_duration = Some(Duration::from_secs(60)); - parameters.initial_delay = Duration::from_secs(60); - let public_config = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); - let (release, gate) = watch::channel(None); - - TransactionGenerator::start_with_gate( - sender, - 0, - parameters, - public_config, - metrics.clone(), - Some(gate), - ); - assert_eq!( - BenchmarkGeneratorState::from_u8( - metrics.benchmark_generator_state.load(Ordering::Acquire) - ), - BenchmarkGeneratorState::Waiting - ); - assert_eq!(metrics.submitted_transactions.get(), 0); - - let start = Instant::now() + Duration::from_millis(20); - let end = start + Duration::from_millis(200); - release.send_replace(BenchmarkTransactionWindow::new(start, end)); - timeout(Duration::from_secs(1), async { - loop { - if BenchmarkGeneratorState::from_u8( - metrics.benchmark_generator_state.load(Ordering::Acquire), - ) == BenchmarkGeneratorState::Finished - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("coordinated generator did not finish its absolute window"); - - let mut received = 0usize; - while let Ok(block) = receiver.try_recv() { - received += block.len(); - } - assert_eq!(received, 4, "ticks must be start, +50, +100, +150 ms"); - assert_eq!(metrics.submitted_transactions.get(), 4); - assert_eq!(metrics.submitted_transactions_bytes.get(), 4 * 512); - assert!(!metrics.metrics_active.load(Ordering::Acquire)); - assert!(metrics.transaction_metrics_active.load(Ordering::Acquire)); - } - - #[tokio::test] - async fn coordinated_window_cancels_a_backpressured_send_at_cutoff() { - let (metrics, _reporter) = Metrics::new(&Registry::new(), None, None, None); - // The first tick fills this channel. The next scheduled pre-end send - // must remain pending until the common cutoff and then be cancelled, - // not counted as offered work after the window. - let (sender, mut receiver) = mpsc::channel(1); - let mut parameters = Parameters::almost_default(20); - parameters.benchmark_duration = Some(Duration::from_secs(60)); - let public_config = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None); - let (release, gate) = watch::channel(None); - TransactionGenerator::start_with_gate( - sender, - 0, - parameters, - public_config, - metrics.clone(), - Some(gate), - ); - - let start = Instant::now() + Duration::from_millis(20); - let end = start + Duration::from_millis(120); - release.send_replace(BenchmarkTransactionWindow::new(start, end)); - timeout(Duration::from_secs(1), async { - loop { - if BenchmarkGeneratorState::from_u8( - metrics.benchmark_generator_state.load(Ordering::Acquire), - ) == BenchmarkGeneratorState::Finished - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("backpressured generator did not finish at the common cutoff"); - - assert_eq!(metrics.submitted_transactions.get(), 1); - assert_eq!(receiver.recv().await.unwrap().len(), 1); - assert!(receiver.try_recv().is_err()); - } } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 2db71d97..a1a75e52 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -11,7 +11,7 @@ use std::{ use ::prometheus::Registry; use eyre::{Context, Result, eyre}; -use tokio::sync::{mpsc, watch}; +use tokio::sync::mpsc; use crate::{ block_handler::{RealBlockHandler, RealCommitHandler}, @@ -19,7 +19,7 @@ use crate::{ config::{NodePrivateConfig, NodePublicConfig, Parameters}, core::Core, dag_state::{DagState, ProtocolConfig}, - metrics::{BenchmarkTransactionWindow, MetricReporter, Metrics}, + metrics::{MetricReporter, Metrics}, net_sync::NetworkSyncer, network::Network, prometheus, @@ -36,18 +36,6 @@ pub struct Validator { reporter: Arc, } -#[derive(Clone, Debug, Default)] -pub struct ValidatorStartOptions { - /// Hold the autonomous RBC-DAG carrier clock until the owner explicitly - /// activates it. This is a local-benchmark coordination primitive, not a - /// serialized validator configuration or a production liveness gate. - pub rbc_dag_clock_start_paused: bool, - /// Optional local-benchmark release latch. Finite transaction generators - /// begin their common warmup only after every coordinated protocol clock - /// has crossed the event-bridge/Core activation barrier. - pub transaction_generator_start: Option>>, -} - impl Validator { pub async fn start( authority: AuthorityIndex, @@ -57,30 +45,6 @@ impl Validator { parameters: Parameters, byzantine_strategy: String, consensus: String, - ) -> Result { - Self::start_with_options( - authority, - committee, - public_config, - private_config, - parameters, - byzantine_strategy, - consensus, - ValidatorStartOptions::default(), - ) - .await - } - - #[allow(clippy::too_many_arguments)] - pub async fn start_with_options( - authority: AuthorityIndex, - committee: Arc, - public_config: NodePublicConfig, - private_config: NodePrivateConfig, - parameters: Parameters, - byzantine_strategy: String, - consensus: String, - start_options: ValidatorStartOptions, ) -> Result { let protocol_config = ProtocolConfig::from_selection( &consensus, @@ -95,16 +59,6 @@ impl Validator { "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" )); } - if start_options.rbc_dag_clock_start_paused - && (!public_config.parameters.starfish_rbc_dag_autonomous_clock - || !public_config - .parameters - .starfish_rbc_dag_embedded_rbc_authority) - { - return Err(eyre!( - "A coordinated RBC-DAG clock start requires autonomous embedded authority" - )); - } if public_config .parameters .starfish_rbc_dag_embedded_rbc_authority @@ -275,13 +229,12 @@ impl Validator { // Rest of the function remains the same let (block_handler, block_sender) = RealBlockHandler::new(&committee); - TransactionGenerator::start_with_gate( + TransactionGenerator::start( block_sender, authority, parameters, public_config.clone(), metrics.clone(), - start_options.transaction_generator_start.clone(), ); let commit_handler = @@ -351,7 +304,6 @@ impl Validator { partial_sig_rx, bls_cert_aggregator, bls_signer_for_service, - start_options.rbc_dag_clock_start_paused, ) .await; @@ -373,17 +325,6 @@ impl Validator { self.reporter.clone() } - pub fn is_finished(&self) -> bool { - self.network_broadcaster.is_finished() || self.metrics_handle.is_finished() - } - - pub async fn activate_starfish_rbc_dag_clock(&self) -> Result<()> { - self.network_broadcaster - .activate_starfish_rbc_dag_clock() - .await - .map_err(|error| eyre!(error)) - } - pub async fn await_completion( self, ) -> ( @@ -397,15 +338,8 @@ impl Validator { } pub async fn stop(self) { - let Self { - network_broadcaster, - metrics_handle, - metrics: _, - reporter: _, - } = self; - metrics_handle.abort(); - let _ = metrics_handle.await; - network_broadcaster.shutdown().await; + self.network_broadcaster.shutdown().await; + self.metrics_handle.abort(); // Give time for background Worker tasks to detect channel closures and exit, // and for TCP sockets to fully release. tokio::time::sleep(std::time::Duration::from_secs(2)).await; @@ -802,40 +736,29 @@ mod smoke_tests { "autonomous mode must not claim direct-round comparison" ); if embedded_rbc_authority { - for message_kind in [ - "rbc_initial", - "rbc_echo", - "rbc_ready", - "rbc_header_request", - "rbc_header_response", - "batch", - "missing_parents", - "missing_tx_data", - ] { - assert_eq!( - metrics - .network_message_bytes_sent_total - .with_label_values(&[message_kind]) - .get(), - 0, - "standalone RBC-DAG mode must not send legacy {message_kind} traffic" - ); - assert_eq!( - metrics - .network_message_bytes_received_total - .with_label_values(&[message_kind]) - .get(), - 0, - "standalone RBC-DAG mode must not receive legacy {message_kind} traffic" - ); - } assert!( metrics .network_message_bytes_sent_total - .with_label_values(&["rbc_dag_shadow_carrier"]) + .with_label_values(&["rbc_initial"]) .get() > 0, - "standalone RBC-DAG carrier transport must be active" + "direct INIT remains the application/payload transport" + ); + assert_eq!( + metrics + .network_message_bytes_sent_total + .with_label_values(&["rbc_echo"]) + .get(), + 0, + "direct RBC ECHO must be disabled under embedded authority" + ); + assert_eq!( + metrics + .network_message_bytes_sent_total + .with_label_values(&["rbc_ready"]) + .get(), + 0, + "direct RBC READY must be disabled under embedded authority" ); } } diff --git a/crates/starfish/Cargo.toml b/crates/starfish/Cargo.toml index 947aea33..27b05bc6 100644 --- a/crates/starfish/Cargo.toml +++ b/crates/starfish/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" clap = { workspace = true } color-eyre = { workspace = true } eyre = { workspace = true } -futures = { workspace = true } prettytable-rs = "0.10" starfish-core = { path = "../starfish-core" } tokio = { workspace = true } diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 24ccf7ac..2f87d35a 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -3,10 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 use std::{ - collections::BTreeSet, fs, - net::{IpAddr, Ipv4Addr, TcpListener, TcpStream}, - path::{Path, PathBuf}, + net::{IpAddr, Ipv4Addr}, + path::PathBuf, sync::{ Arc, atomic::{AtomicBool, AtomicU64, Ordering}, @@ -17,29 +16,21 @@ use std::{ use clap::Parser; use eyre::{Context, Result}; -use futures::future::try_join_all; use prettytable::format; use starfish_core::{ + ByzantineStrategy, committee::Committee, config::{ DisseminationMode, ImportExport, NodeParameters, NodePrivateConfig, NodePublicConfig, Parameters, StorageBackend, TransactionMode, }, - metrics::{ - AutonomousClockBenchmarkSnapshot, BenchmarkGeneratorState, BenchmarkTransactionWindow, - LocalBenchmarkTransactionOutcome, Metrics, - }, + metrics::Metrics, types::AuthorityIndex, - validator::{Validator, ValidatorStartOptions}, + validator::Validator, }; -use tokio::{sync::watch, time::Instant}; +use tokio::time::Instant; use tracing_subscriber::{EnvFilter, filter::LevelFilter, fmt}; -// Network workers bind active sockets at `listener_port * 10`. Keep those -// derived ports below the conventional Linux ephemeral range so repeated -// local experiments cannot collide with an unrelated outbound connection. -const LOCAL_BENCHMARK_MAX_ACTIVE_BIND_PORT: u16 = 32_767; - #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Args { @@ -217,11 +208,6 @@ enum Operation { starfish_rbc_dag_shadow_buffered_wal: bool, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, - /// Add an offset to the local benchmark's fixed network and metrics - /// ports. This is useful for isolated repeated experiments and avoids - /// silently sharing `SO_REUSEPORT` listeners with a stale run. - #[clap(long, value_name = "INT", default_value_t = 0)] - port_offset: u16, /// Dissemination mode override: /// protocol-default | pull | push-causal | push-useful #[clap(long, value_name = "STRING")] @@ -336,7 +322,6 @@ async fn main() -> Result<()> { starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_shadow_buffered_wal, duration_secs, - port_offset, dissemination_mode, } => { let mut node_parameters = NodeParameters::default_with_latency(mimic_extra_latency); @@ -368,7 +353,6 @@ async fn main() -> Result<()> { node_parameters, consensus_protocol, duration_secs, - port_offset, ) .await?; } @@ -436,116 +420,14 @@ fn benchmark_genesis( Ok(()) } -fn local_benchmark_private_configs( - base_dir: &Path, - committee_size: usize, -) -> Vec { - NodePrivateConfig::new_for_benchmarks(base_dir, committee_size) - .into_iter() - .enumerate() - .map(|(authority, mut private_config)| { - private_config.storage_path = base_dir.join(format!("node-{authority}")).join( - NodePrivateConfig::default_storage_path(authority as AuthorityIndex), - ); - private_config - }) - .collect() -} - -async fn stop_local_benchmark_validators(validators: Vec) { - let mut stops = tokio::task::JoinSet::new(); - for validator in validators { - stops.spawn(validator.stop()); - } - while let Some(result) = stops.join_next().await { - if let Err(error) = result { - tracing::warn!("Validator shutdown task failed: {error}"); - } - } -} - -fn local_benchmark_topology_ready( - metrics: &[Arc], - expected_peer_subscriptions: i64, -) -> Result { - local_benchmark_topology_state( - metrics.iter().map(|metrics| { - ( - metrics.metrics_active.load(Ordering::Relaxed), - metrics.subscribed_to_peers.get(), - metrics.subscribed_by_peers.get(), - ) - }), - expected_peer_subscriptions, - ) -} - -fn local_benchmark_topology_state( - states: impl IntoIterator, - expected_peer_subscriptions: i64, -) -> Result { - let states = states.into_iter().collect::>(); - eyre::ensure!( - !states.iter().any(|(active, _, _)| *active), - "A transaction generator opened before the complete peer subscription mesh" - ); - Ok(states.iter().all(|(_, subscribed_to, subscribed_by)| { - *subscribed_to == expected_peer_subscriptions - && *subscribed_by == expected_peer_subscriptions - })) -} - -/// Assign the configured aggregate transaction load exclusively to honest -/// validators. Byzantine validators exercise protocol behavior only: making -/// their deliberately withheld/dropped transactions part of the mandatory -/// drain target would conflate adversarial dissemination with lost honest -/// work. The remainder is distributed deterministically by authority order so -/// the per-validator rates sum to the exact requested aggregate load. -fn local_benchmark_generator_loads( - committee_size: usize, - aggregate_load: usize, - num_byzantine_nodes: usize, -) -> Result<(BTreeSet, Vec)> { - let byzantine_authorities = (0..committee_size) - .filter(|authority| *authority % 3 == 0 && *authority / 3 < num_byzantine_nodes) - .collect::>(); - eyre::ensure!( - byzantine_authorities.len() == num_byzantine_nodes, - "requested {num_byzantine_nodes} Byzantine validators, but the local benchmark layout can place only {} in a committee of {committee_size}", - byzantine_authorities.len(), - ); - let honest_count = committee_size.saturating_sub(byzantine_authorities.len()); - eyre::ensure!( - honest_count > 0, - "a local benchmark requires at least one honest transaction generator" - ); - let base = aggregate_load / honest_count; - let remainder = aggregate_load % honest_count; - let mut honest_rank = 0usize; - let loads = (0..committee_size) - .map(|authority| { - if byzantine_authorities.contains(&authority) { - 0 - } else { - let load = base + usize::from(honest_rank < remainder); - honest_rank += 1; - load - } - }) - .collect::>(); - debug_assert_eq!(loads.iter().sum::(), aggregate_load); - Ok((byzantine_authorities, loads)) -} - async fn local_benchmark( committee_size: usize, - load: usize, + mut load: usize, num_byzantine_nodes: usize, byzantine_strategy: String, node_parameters: NodeParameters, consensus_protocol: String, duration_secs: u64, - port_offset: u16, ) -> Result<()> { eyre::ensure!( duration_secs > 0, @@ -612,52 +494,34 @@ async fn local_benchmark( ); } println!("Duration: {duration_secs} seconds"); - println!("Local port offset: {port_offset}"); println!("===========================\n"); let ips = vec![IpAddr::V4(Ipv4Addr::LOCALHOST); committee_size]; let committee = Committee::new_for_benchmarks(committee_size); - let (byzantine_authorities, generator_loads) = - local_benchmark_generator_loads(committee_size, load, num_byzantine_nodes)?; - if !byzantine_authorities.is_empty() { - println!( - "Byzantine transaction generators: disabled; aggregate load redistributed across {} honest validators", - committee_size.saturating_sub(byzantine_authorities.len()), - ); - } - let mut parameters = Parameters::almost_default(0); + load /= committee.len(); + let mut parameters = Parameters::almost_default(load); parameters.benchmark_duration = Some(Duration::from_secs(duration_secs)); + // Equivocating Byzantine strategies must not generate transactions. + let mut byzantine_parameters = parameters.clone(); + if ByzantineStrategy::from_strategy_str(&byzantine_strategy) + .is_some_and(|s| s.is_equivocating()) + { + byzantine_parameters.load = 0; + } let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); - validate_local_benchmark_port_offset(&public_config, port_offset)?; - let public_config = public_config.with_port_offset(port_offset); - preflight_local_benchmark_ports(&public_config)?; let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; let starfish_rbc_dag_autonomous_clock_expected = node_parameters.starfish_rbc_dag_autonomous_clock; let starfish_rbc_dag_embedded_rbc_authority_expected = node_parameters.starfish_rbc_dag_embedded_rbc_authority; - let coordinated_rbc_dag_clock_start = starfish_rbc_dag_autonomous_clock_expected - && starfish_rbc_dag_embedded_rbc_authority_expected; // Create temporary directories for each validator - // Isolate storage by the same explicit run namespace as the sockets. - // A stale or intentionally concurrent benchmark on another offset must - // not keep RocksDB/WAL handles open underneath this run's cleanup. - let base_dir = PathBuf::from(format!("local-benchmark-{port_offset}")); + let base_dir = PathBuf::from("local-benchmark"); fs::create_dir_all(&base_dir)?; - // Generate the benchmark key material before any validator starts. Doing - // this inside the startup loop regenerates the entire committee keyset for - // every authority; once the first quorum is live, that CPU work lets its - // autonomous clock run far ahead of the validators still being prepared. - let private_configs = local_benchmark_private_configs(&base_dir, committee_size); - let mut validators = Vec::with_capacity(committee_size); - let mut metrics_of_all_validators = Vec::with_capacity(committee_size); + let mut handles = Vec::with_capacity(committee_size); + let mut abort_handles = Vec::with_capacity(committee_size); let mut metrics_of_honest_validators = Vec::new(); let mut reporters_of_honest_validators = Vec::new(); - // Every local benchmark, not only RBC-DAG, uses one absolute offered-load - // window. Production Validator::start remains ungated. - let (transaction_generator_start_tx, transaction_generator_start_rx) = - watch::channel(None::); // Create a flag to signal when the benchmark is complete let running = Arc::new(AtomicBool::new(true)); @@ -668,7 +532,7 @@ async fn local_benchmark( run_with_progress(running.clone(), elapsed_seconds.clone()); // Start all validators - for (authority, private_config) in private_configs.into_iter().enumerate() { + for authority in 0..committee_size { tracing::warn!( "Starting node {authority} in local \ benchmark mode (committee size: {committee_size})" @@ -685,6 +549,9 @@ async fn local_benchmark( )); } } + let mut private_configs = + NodePrivateConfig::new_for_benchmarks(&working_dir, committee_size); + let private_config = private_configs.remove(authority); match fs::create_dir_all(&private_config.storage_path) { Ok(_) => {} Err(e) => { @@ -694,49 +561,53 @@ async fn local_benchmark( )); } } - let is_byzantine = byzantine_authorities.contains(&authority); - let mut generator_parameters = parameters.clone(); - generator_parameters.load = generator_loads[authority]; - let start_options = ValidatorStartOptions { - rbc_dag_clock_start_paused: coordinated_rbc_dag_clock_start, - transaction_generator_start: Some(transaction_generator_start_rx.clone()), - }; + let is_byzantine = authority.is_multiple_of(3) && authority / 3 < num_byzantine_nodes; let validator = if is_byzantine { - Validator::start_with_options( + Validator::start( authority as AuthorityIndex, committee.clone(), public_config.clone(), private_config, - generator_parameters, + byzantine_parameters.clone(), byzantine_strategy.clone(), consensus_protocol.clone(), - start_options, ) .await? } else { - Validator::start_with_options( + Validator::start( authority as AuthorityIndex, committee.clone(), public_config.clone(), private_config, - generator_parameters, + parameters.clone(), "honest".to_string(), consensus_protocol.clone(), - start_options, ) .await? }; let validator_metrics = validator.metrics(); - metrics_of_all_validators.push(Arc::clone(&validator_metrics)); if !is_byzantine { metrics_of_honest_validators.push(Arc::clone(&validator_metrics)); reporters_of_honest_validators.push(validator.reporter()) } - validators.push(validator); + // Use the same pattern as the run method + let handle = tokio::spawn(async move { + let (network_result, _metrics_result) = validator.await_completion().await; + if starfish_rbc_dag_autonomous_clock_expected { + validator_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); + } else if starfish_rbc_dag_shadow_expected { + validator_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); + } + network_result + }); + abort_handles.push(handle.abort_handle()); + handles.push(handle); } - if starfish_rbc_dag_shadow_expected && !coordinated_rbc_dag_clock_start { + if starfish_rbc_dag_shadow_expected { let ready = tokio::time::timeout(Duration::from_secs(30), async { loop { if metrics_of_honest_validators.iter().all(|metrics| { @@ -753,8 +624,9 @@ async fn local_benchmark( }) .await; if ready.is_err() { - running.store(false, Ordering::SeqCst); - stop_local_benchmark_validators(validators).await; + for abort_handle in &abort_handles { + abort_handle.abort(); + } fs::remove_dir_all(&base_dir)?; let mode = if starfish_rbc_dag_autonomous_clock_expected { "autonomous clock" @@ -767,110 +639,26 @@ async fn local_benchmark( } } - // A ready protocol actor is not yet a ready benchmark network. Require - // every honest validator to observe the complete logical subscription - // mesh before recording baselines. This fails closed if an earlier core - // failure leaves independent TCP workers alive or if startup is only - // partially connected. - let expected_peer_subscriptions = - i64::try_from(committee_size.saturating_sub(1)).unwrap_or(i64::MAX); - let topology_ready = tokio::time::timeout(Duration::from_secs(30), async { + // `duration_secs` is an active transaction-submission window, not a + // process-lifetime cutoff. Every finite generator holds metrics inactive + // through connection warmup, then opens this latch immediately before its + // first batch. Start the benchmark only after every honest validator has + // crossed that boundary. + tokio::time::timeout(Duration::from_secs(30), async { loop { - if local_benchmark_topology_ready( - &metrics_of_all_validators, - expected_peer_subscriptions, - )? { + if metrics_of_honest_validators + .iter() + .all(|metrics| metrics.metrics_active.load(Ordering::Relaxed)) + { break; } tokio::time::sleep(Duration::from_millis(25)).await; } - Ok::<(), eyre::Report>(()) }) - .await; - if !matches!(topology_ready, Ok(Ok(()))) { - running.store(false, Ordering::SeqCst); - stop_local_benchmark_validators(validators).await; - fs::remove_dir_all(&base_dir)?; - eyre::bail!( - "Local benchmark did not establish the complete {expected_peer_subscriptions}-peer subscription mesh on every honest validator" - ); - } - - if coordinated_rbc_dag_clock_start { - let activated = tokio::time::timeout( - Duration::from_secs(30), - try_join_all( - validators - .iter() - .map(Validator::activate_starfish_rbc_dag_clock), - ), - ) - .await; - if let Ok(Err(error)) = &activated { - tracing::error!("Failed to activate a coordinated RBC-DAG clock: {error}"); - } - if !matches!(activated, Ok(Ok(_))) { - running.store(false, Ordering::SeqCst); - stop_local_benchmark_validators(validators).await; - fs::remove_dir_all(&base_dir)?; - eyre::bail!( - "Local benchmark could not activate every RBC-DAG clock after full topology readiness" - ); - } - - // The service activation acknowledgment only publishes the ordered - // ClockActivated event. Wait until each event bridge has processed it, - // released the authoritative Core producer, and marked the clock live. - let clock_ready = tokio::time::timeout(Duration::from_secs(30), async { - loop { - local_benchmark_topology_ready( - &metrics_of_all_validators, - expected_peer_subscriptions, - )?; - if metrics_of_all_validators - .iter() - .all(|metrics| metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1) - { - break; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - Ok::<(), eyre::Report>(()) - }) - .await; - if !matches!(clock_ready, Ok(Ok(()))) { - running.store(false, Ordering::SeqCst); - stop_local_benchmark_validators(validators).await; - fs::remove_dir_all(&base_dir)?; - eyre::bail!("RBC-DAG clocks did not activate before the transaction window opened"); - } - } - - let generators_waiting = tokio::time::timeout(Duration::from_secs(5), async { - loop { - if metrics_of_all_validators.iter().all(|metrics| { - BenchmarkGeneratorState::from_u8( - metrics.benchmark_generator_state.load(Ordering::Acquire), - ) == BenchmarkGeneratorState::Waiting - }) { - break; - } - tokio::task::yield_now().await; - } - }) - .await; - if generators_waiting.is_err() { - running.store(false, Ordering::SeqCst); - stop_local_benchmark_validators(validators).await; - fs::remove_dir_all(&base_dir)?; - eyre::bail!("transaction generators did not reach the coordinated waiting state"); - } + .await + .wrap_err("transaction generators did not open the active benchmark window")?; + println!("Active transaction window started ({duration_secs} seconds)"); - // Clear warmup samples and capture every baseline before publishing the - // release. No generator can submit before the common absolute start. - for reporter in &reporters_of_honest_validators { - reporter.reset_for_benchmark_window(); - } let autonomous_clock_baselines = starfish_rbc_dag_autonomous_clock_expected.then(|| { metrics_of_honest_validators .iter() @@ -881,263 +669,59 @@ async fn local_benchmark( .iter() .map(|metrics| metrics.local_benchmark_counter_baseline()) .collect::>(); - let sequenced_baselines = metrics_of_honest_validators - .iter() - .map(|metrics| metrics.sequenced_transactions_total.get()) - .collect::>(); - let cutoff_sequenced_baselines = metrics_of_honest_validators - .iter() - .map(|metrics| metrics.sequenced_transactions_cutoff_total.get()) - .collect::>(); - let honest_submitted_baselines = metrics_of_honest_validators - .iter() - .map(|metrics| metrics.submitted_transactions.get()) - .collect::>(); - let window_start = Instant::now() + Duration::from_millis(100); - let window_end = window_start - .checked_add(Duration::from_secs(duration_secs)) - .ok_or_else(|| eyre::eyre!("benchmark duration exceeds the monotonic clock range"))?; - let transaction_window = BenchmarkTransactionWindow::new(window_start, window_end) - .expect("positive local benchmark duration must form a valid window"); - for metrics in &metrics_of_all_validators { - let cutoff_micros = window_end - .saturating_duration_since(metrics.validator_start) - .as_micros() - .min(u64::MAX as u128) as u64; - metrics - .benchmark_transaction_cutoff_micros - .store(cutoff_micros, Ordering::Release); - } - transaction_generator_start_tx.send_replace(Some(transaction_window)); - tokio::time::sleep_until(window_start).await; - let active_window_ready = tokio::time::timeout(Duration::from_secs(5), async { - loop { - let states = metrics_of_all_validators - .iter() - .map(|metrics| { - BenchmarkGeneratorState::from_u8( - metrics.benchmark_generator_state.load(Ordering::Acquire), - ) - }) - .collect::>(); - eyre::ensure!( - !states.contains(&BenchmarkGeneratorState::Failed), - "a transaction generator failed while opening the common window" + // Run for specified duration + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(duration_secs)) => { + // Signal the progress display to stop + running.store(false, Ordering::SeqCst); + println!(); + println!("Benchmark completed after {duration_secs} seconds"); + // Display metrics + Metrics::aggregate_and_display( + metrics_of_honest_validators, + reporters_of_honest_validators, + duration_secs, + committee_size, + starfish_rbc_dag_shadow_expected, + starfish_rbc_dag_autonomous_clock_expected, + starfish_rbc_dag_embedded_rbc_authority_expected, + autonomous_clock_baselines.clone(), + Some(counter_baselines.clone()), ); - if states - .iter() - .all(|state| *state == BenchmarkGeneratorState::Active) - { - break; - } - tokio::task::yield_now().await; - } - Ok::<(), eyre::Report>(()) - }) - .await; - if !matches!(active_window_ready, Ok(Ok(()))) { - running.store(false, Ordering::SeqCst); - stop_local_benchmark_validators(validators).await; - fs::remove_dir_all(&base_dir)?; - eyre::bail!("transaction generators did not open the common active benchmark window"); - } - println!("Active transaction window started ({duration_secs} seconds)"); - // Run for the requested active duration, but fail if any validator exits. - // Keep ownership of the validators so the normal path can use their - // graceful shutdown rather than aborting wrapper tasks and leaking socket - // workers into subsequent latency experiments. - let completed_full_duration = tokio::select! { - _ = tokio::time::sleep_until(window_end) => true, - _ = async { - loop { - if validators.iter().any(Validator::is_finished) { - break; - } - tokio::time::sleep(Duration::from_millis(25)).await; + // Abort all tasks + for abort_handle in abort_handles { + abort_handle.abort(); } - } => false, - }; - // Close protocol/window metrics at the exact common cutoff even if a - // delayed generator task still has a scheduled pre-end batch to publish. - for metrics in &metrics_of_all_validators { - metrics.metrics_active.store(false, Ordering::Release); - } - let autonomous_clock_cutoffs: Option> = - starfish_rbc_dag_autonomous_clock_expected.then(|| { - metrics_of_honest_validators - .iter() - .map(|metrics| metrics.autonomous_clock_benchmark_snapshot()) - .collect() - }); - let counter_cutoffs = metrics_of_honest_validators - .iter() - .map(|metrics| metrics.local_benchmark_counter_baseline()) - .collect::>(); - let cutoff_committed_transactions = metrics_of_honest_validators - .iter() - .zip(&cutoff_sequenced_baselines) - .map(|(metrics, baseline)| { - metrics - .sequenced_transactions_cutoff_total - .get() - .saturating_sub(*baseline) - }) - .sum::() - / metrics_of_honest_validators.len() as u64; - - let drain_started = Instant::now(); - let generators_finished = completed_full_duration - && matches!( - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let states = metrics_of_all_validators - .iter() - .map(|metrics| { - BenchmarkGeneratorState::from_u8( - metrics.benchmark_generator_state.load(Ordering::Acquire), - ) - }) - .collect::>(); - eyre::ensure!( - !states.contains(&BenchmarkGeneratorState::Failed), - "a transaction generator failed during the active window" - ); - if states - .iter() - .all(|state| *state == BenchmarkGeneratorState::Finished) - { - break; - } - tokio::task::yield_now().await; - } - Ok::<(), eyre::Report>(()) - }) - .await, - Ok(Ok(())) - ); - - // Byzantine generators have zero configured load, and offered work is - // defined explicitly from honest generators so adversarial strategies can - // neither inflate nor make the mandatory drain target unattainable. - let offered_transactions = metrics_of_honest_validators - .iter() - .zip(&honest_submitted_baselines) - .map(|(metrics, baseline)| { - metrics - .submitted_transactions - .get() - .saturating_sub(*baseline) - }) - .sum::(); - let pipeline_observed_through_target = |metrics: &Metrics, sequenced_baseline: u64| { - !starfish_rbc_dag_embedded_rbc_authority_expected - || offered_transactions == 0 - || metrics.starfish_rbc_dag_frontier_applied_sequenced_transactions() - >= sequenced_baseline.saturating_add(offered_transactions) - }; - let drain_complete = generators_finished - && tokio::time::timeout(Duration::from_secs(30), async { - loop { - if metrics_of_honest_validators - .iter() - .zip(&sequenced_baselines) - .all(|(metrics, baseline)| { - let sequenced = metrics - .sequenced_transactions_total - .get() - .saturating_sub(*baseline); - sequenced >= offered_transactions - && pipeline_observed_through_target(metrics, *baseline) - }) - { - break; - } - if validators.iter().any(Validator::is_finished) { - break; + // Clean up + fs::remove_dir_all(base_dir)?; + Ok(()) + } + _ = async { + for handle in handles { + if let Err(e) = handle.await { + tracing::warn!("Validator terminated with error: {}", e); } - tokio::time::sleep(Duration::from_millis(10)).await; } - }) - .await - .is_ok() - && metrics_of_honest_validators - .iter() - .zip(&sequenced_baselines) - .all(|(metrics, baseline)| { - metrics - .sequenced_transactions_total - .get() - .saturating_sub(*baseline) - == offered_transactions - }) - && metrics_of_honest_validators - .iter() - .zip(&sequenced_baselines) - .all(|(metrics, baseline)| pipeline_observed_through_target(metrics, *baseline)); - let drain_elapsed = drain_started.elapsed(); - for metrics in &metrics_of_all_validators { - metrics - .transaction_metrics_active - .store(false, Ordering::Release); - } - let eventual_committed_transactions = metrics_of_honest_validators - .iter() - .zip(&sequenced_baselines) - .map(|(metrics, baseline)| { - metrics - .sequenced_transactions_total - .get() - .saturating_sub(*baseline) - }) - .sum::() - / metrics_of_honest_validators.len() as u64; - let transaction_outcome = LocalBenchmarkTransactionOutcome { - offered_transactions, - cutoff_committed_transactions, - eventual_committed_transactions, - drain_elapsed, - drain_complete, - }; - - running.store(false, Ordering::SeqCst); - println!(); - if completed_full_duration { - println!("Benchmark completed after {duration_secs} seconds"); - } else { - println!("A validator completed before timeout"); + } => { + println!("All validators completed before timeout"); + Metrics::aggregate_and_display( + metrics_of_honest_validators, + reporters_of_honest_validators, + duration_secs, + committee_size, + starfish_rbc_dag_shadow_expected, + starfish_rbc_dag_autonomous_clock_expected, + starfish_rbc_dag_embedded_rbc_authority_expected, + autonomous_clock_baselines, + Some(counter_baselines), + ); + fs::remove_dir_all(base_dir)?; + eyre::bail!("All validators completed before the requested benchmark duration") + } } - Metrics::aggregate_and_display( - metrics_of_honest_validators, - reporters_of_honest_validators, - duration_secs, - committee_size, - starfish_rbc_dag_shadow_expected, - starfish_rbc_dag_autonomous_clock_expected, - starfish_rbc_dag_embedded_rbc_authority_expected, - autonomous_clock_baselines, - autonomous_clock_cutoffs, - Some(counter_baselines), - Some(counter_cutoffs), - Some(transaction_outcome), - ); - stop_local_benchmark_validators(validators).await; - fs::remove_dir_all(base_dir)?; - eyre::ensure!( - completed_full_duration, - "A validator completed before the requested benchmark duration" - ); - eyre::ensure!( - generators_finished, - "A transaction generator failed to finish the common active window" - ); - eyre::ensure!( - drain_complete, - "Active-window transaction drain was incomplete: offered={offered_transactions}, eventual average committed={eventual_committed_transactions}" - ); - Ok(()) } /// Boot a single validator node. @@ -1380,7 +964,6 @@ fn preflight_local_benchmark_ports(public_config: &NodePublicConfig) -> Result<( } Ok(()) } - fn ipv4_add_offset(base: Ipv4Addr, offset: usize) -> Result { let offset = u32::try_from(offset).context("validator count exceeds IPv4 offset range")?; let next = u32::from(base) @@ -1438,23 +1021,12 @@ pub fn default_table_format() -> format::TableFormat { #[cfg(test)] mod tests { - use std::{ - net::{IpAddr, Ipv4Addr, TcpListener}, - path::PathBuf, - }; + use std::net::Ipv4Addr; use clap::Parser; - use super::{ - Args, Operation, ensure_starfish_rbc_protocol_instance, ipv4_add_offset, - local_benchmark_generator_loads, local_benchmark_private_configs, - local_benchmark_topology_state, preflight_local_benchmark_ports, - validate_local_benchmark_port_offset, - }; - use starfish_core::{ - config::{NodeParameters, NodePrivateConfig, NodePublicConfig}, - types::AuthorityIndex, - }; + use super::{Args, Operation, ensure_starfish_rbc_protocol_instance, ipv4_add_offset}; + use starfish_core::config::NodeParameters; #[test] fn ipv4_add_offset_crosses_octet_boundary() { @@ -1471,22 +1043,6 @@ mod tests { assert!(ipv4_add_offset(base, 1).is_err()); } - #[test] - fn local_benchmark_disables_byzantine_generators_and_preserves_aggregate_load() { - let (byzantine, loads) = local_benchmark_generator_loads(10, 1_003, 2).unwrap(); - - assert_eq!(byzantine.into_iter().collect::>(), vec![0, 3]); - assert_eq!(loads[0], 0); - assert_eq!(loads[3], 0); - assert_eq!(loads.iter().sum::(), 1_003); - assert_eq!(loads, vec![0, 126, 126, 0, 126, 125, 125, 125, 125, 125]); - } - - #[test] - fn local_benchmark_rejects_unplaceable_byzantine_count() { - assert!(local_benchmark_generator_loads(4, 100, 3).is_err()); - } - #[test] fn dry_run_parses_block_authentication_separately_from_consensus() { let args = Args::try_parse_from([ @@ -1531,8 +1087,6 @@ mod tests { "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-single-dag-echo-qc-fast-path", "--starfish-rbc-dag-shadow-buffered-wal", - "--port-offset", - "2500", ]) .unwrap(); @@ -1544,7 +1098,6 @@ mod tests { starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_shadow_buffered_wal, - port_offset, .. } = args.operation else { @@ -1557,7 +1110,6 @@ mod tests { assert!(starfish_rbc_dag_embedded_rbc_authority); assert!(starfish_rbc_single_dag_echo_qc_fast_path); assert!(starfish_rbc_dag_shadow_buffered_wal); - assert_eq!(port_offset, 2500); } #[test] @@ -1578,56 +1130,4 @@ mod tests { assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); } - - #[test] - fn local_benchmark_port_offset_stays_below_ephemeral_active_ports() { - let safe = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST); 10], None); - validate_local_benchmark_port_offset(&safe, 200).unwrap(); - - let ephemeral = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST); 10], None); - assert!(validate_local_benchmark_port_offset(&ephemeral, 3_500).is_err()); - assert!(validate_local_benchmark_port_offset(&ephemeral, u16::MAX).is_err()); - } - - #[test] - fn local_benchmark_private_configs_preserve_per_authority_storage_layout() { - let base_dir = PathBuf::from("local-benchmark-config-test"); - let committee_size = 4; - let private_configs = local_benchmark_private_configs(&base_dir, committee_size); - - assert_eq!(private_configs.len(), committee_size); - for (authority, private_config) in private_configs.into_iter().enumerate() { - assert_eq!(private_config.mac_keys.len(), committee_size); - assert_eq!( - private_config.storage_path, - base_dir.join(format!("node-{authority}")).join( - NodePrivateConfig::default_storage_path(authority as AuthorityIndex) - ) - ); - } - } - - #[test] - fn local_benchmark_preflight_rejects_an_existing_listener() { - let public_config = - NodePublicConfig::new_for_benchmarks(vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], None) - .with_port_offset(1_400); - let address = public_config.network_address(0).unwrap(); - let _listener = match TcpListener::bind(address) { - Ok(listener) => listener, - Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => return, - Err(error) => panic!("failed to bind preflight test listener: {error}"), - }; - - assert!(preflight_local_benchmark_ports(&public_config).is_err()); - } - - #[test] - fn local_benchmark_topology_readiness_fails_closed_before_baselining() { - assert!(!local_benchmark_topology_state([(false, 2, 3)], 3).unwrap()); - assert!(local_benchmark_topology_state([(false, 3, 3)], 3).unwrap()); - assert!(local_benchmark_topology_state([(true, 3, 3)], 3).is_err()); - } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index cf5bcce5..3cbcc83a 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -10,22 +10,25 @@ Status: standalone MAC-vector RBC-DAG prototype with authoritative optimistic de committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoint transfer, and full validator recovery remain incomplete -The provisional CLI name for the eventual protocol is `starfish-rbc-dag`; that selector is not yet -implemented. The prototype runs under `starfish-rbc`: direct-header comparison uses -`--starfish-rbc-dag-shadow`, and standalone authority additionally enables -`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority`. In standalone -mode the direct Starfish-RBC service is not started. Direct INIT, direct phase messages, direct -header pull, generic block batches, legacy missing-parent pull, and legacy transaction-data pull -have no certification, consensus, ordering, or output authority. Canonical application headers are -inside carriers; optional application bytes use the carrier envelope or the dedicated RBC-DAG -payload request/response path. Committed projected anchors release cumulative exact -carrier-frontier deltas, and those deltas are the sole application-ordering/output authority. - -The implemented direct [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains a separate -historical baseline: it sends INIT/ECHO/READY as direct messages and advances Starfish only through -its direct delivery path. This document specifies the standalone optimistic carrier DAG, its -four-phase weighted ECHO/VOTE/ACK/READY broadcast, its complete committee-sized MAC-vector -sidecar, and the separation between fast carrier pacing and logical consensus ordering. +The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not +implemented. The staged prototype runs under `starfish-rbc`: direct-header comparison uses +`--starfish-rbc-dag-shadow`, the independent carrier clock adds +`--starfish-rbc-dag-autonomous-clock`, and milestone five makes embedded carrier ECHO/READY the +only application-header certification authority with +`--starfish-rbc-dag-embedded-rbc-authority`. Direct INIT still transports the application payload, +but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add +`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Autonomous +carriers now create durably locked logical consensus vertices and the clean projection produces +Starfish commit/skip decisions. In embedded-authority mode, committed projected anchors now release +deterministic exact carrier-frontier deltas and the legacy Starfish committer is disabled. The +eventual protocol is new, not a transport option or a version-two alias for `starfish-rbc`. + +The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative +baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through +RBC-delivered dependency-closed headers, and sends one initial MAC tag to each recipient. This +document instead specifies an optimistic carrier DAG that embeds the reliable-broadcast transcript, +uses a complete committee-sized MAC vector on every MAC-authenticated carrier, and separates fast +carrier pacing from certified consensus ordering. The two designs may share canonicalization, cryptography, storage, and benchmark code, but they are not wire-compatible and do not have the same proof obligations. Nothing in this document changes @@ -37,10 +40,9 @@ The objective is to recover the pipelining of an uncertified Starfish DAG withou optimistically received Byzantine blocks to affect safety: - a fast physical carrier DAG advances on a quorum of locally authenticated carriers; -- every carrier is the exact value of a weighted reliable-broadcast slot; -- later carriers batch ECHO, VOTE, ACK, and READY statements for earlier carriers; -- only authoritatively delivered, data-available exact prefixes enter the logical consensus - projection; and +- every application carrier is also the value of a Bracha reliable-broadcast instance; +- later carriers batch ECHO and READY statements for earlier carriers; +- only RBC-delivered, data-available information enters the logical consensus projection; and - committed consensus frontiers eventually order every honest on-prefix application carrier. The initial research mode sends the complete ordered MAC vector with every carrier. The vector is @@ -48,33 +50,50 @@ an authentication sidecar and is not part of the carrier digest. Ed25519, ML-DSA remain selectable outer-authentication baselines; changing that selector does not change the embedded RBC or consensus rules. -The implemented composition includes the canonical codec, deterministic four-phase reducer, -durable proof-critical journal, autonomous carrier clock, exact carrier recovery, optimistic and -certification latches, consensus projection, and cumulative committed-frontier output. Idle control -heartbeats use the resolved Starfish leader timeout (600 ms for Push/Starfish-RBC by default), but -application carriers and encodable phase follow-ups are event-driven. The V4 authoritative mode -promotes the proved `O`-ECHO predicate to delivery authority; `Q` READY remains a distinct slower -certification fact rather than a prerequisite for fast projection. - -Standalone authority is an end-to-end boundary. The direct Starfish-RBC actor is absent, generic -block dissemination and legacy pull messages are rejected or ignored, and the legacy Starfish -committer is disabled. Only durably authoritatively delivered carrier content, the typed -verified-payload ingress, projected consensus decisions, and committed frontier deltas can affect -application output. - -Restart coverage is deliberately scoped. The actor WAL replays retained exact content, ordered -ingress, local ECHO/VOTE/ACK/READY locks, delivery/promise locks, local outbound bytes, consensus -choices, projection state, and committed frontiers. Exact-slot responses are byte-identical when -the requested locally authored outbound carrier remains retained. This is not yet a full validator -crash-recovery, checkpoint-transfer, proof-safe retirement, or arbitrary late-node catch-up claim. - -The carrier plane shares the validator's existing TCP connections, outbound queues, bandwidth, and -CPU process. There is no feature handshake, so every validator in a run must understand the -append-only carrier messages and enable a homogeneous configuration. Mirror and autonomous modes -derive distinct authentication protocol instances. The authoritative durable path additionally -uses the V4 autonomous WAL filename and `SRD5` raw-record magic, preventing older promise semantics -or record layouts from being silently replayed as the current protocol. These fail-closed -namespaces are not a substitute for deployment negotiation. +This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish +commit rules already exist. Milestone two provides a canonical codec plus deterministic +carrier/RBC, certified-projection, decision, and crash-journal models. Milestone three adds an +opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired +direct/shadow delivery observations. Milestone four adds an independently authenticated autonomous +heartbeat namespace, sequential `Q`-admitted carrier clock, bounded future buffering, exact-slot +synchronization, and clock-state metrics. Milestone five adds a version-two application carrier +containing the exact canonical application header, durable application-origin reconciliation, +immediate application/phase scheduling, and an opt-in authority boundary that prevents direct +ECHO/READY from certifying a header. Idle control heartbeats use the same resolved leader timeout as +the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application carriers and their +encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. Milestone six +adds independently numbered consensus vertices, quorum strong parents, objective Vote/NoVote +choices, exact contiguous delivery frontiers, durable local consensus locks, and a live committer +that consumes only RBC-delivered, data-available projected vertices. Milestone seven persists and +reconstructs the corresponding anchor/frontier state, applies exact closed-prefix deltas, and makes +those deltas the sole application output authority. + +The current authoritative mode changes header certification and application ordering. Direct INIT +is still the application-payload transport and is not a certification vote. Direct ECHO/READY and +the legacy Starfish committer cannot certify or output applications in this mode; only the +carrier-DAG projection's committed frontier deltas can do so. + +Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode +requires an identical recovered direct-header history, control-only autonomous history reopens +without direct headers, and autonomous application origins must match recovered direct history. +Every mode serves byte-identical exact-slot responses. This is not a full validator crash-recovery +claim. The authoritative direct +`starfish-rbc` baseline does not yet durably record its remote-slot +ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it +has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest +recovered own header dirty. Full validator restart remains fail-stop until those direct-RBC locks +are persisted and replayed; replaying only the observational shadow WAL cannot repair or safely +substitute for them. + +That isolation is logical, not physical: shadow frames share the validator's existing TCP +connections, outbound queues, bandwidth, and CPU process with direct RBC, so enabling the shadow +can perturb authoritative timing even though no shadow result is consumed by consensus. Version +one has no feature handshake. Every validator in a shadow run must use a binary that understands +the append-only shadow wire variants and the flag must be deployed committee-wide; an older peer +will reject an unknown bincode variant and may close the shared connection. Mixed-version or +partially enabled runs are not valid comparisons. Mirror and autonomous carriers derive distinct +authentication protocol instances so they cannot cross-admit, but this fail-closed boundary is not +a substitute for capability negotiation. Shadow shutdown is bounded so an observational WAL failure cannot indefinitely block validator shutdown. If that timeout fires, the blocking worker may still hold the shadow WAL's single-writer @@ -92,17 +111,19 @@ separately and makes no crash-safety claim. This removes the known persistence o without changing the protocol reducer. The runtime also uses a fixed unsolicited-retention window only as a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the composition and resource bounds are completed, `starfish-rbc-dag` remains an experimental -reference implementation rather than a proven signature-free Starfish variant. +shadow/reference implementation rather than a proven signature-free Starfish variant. -Application bytes remain outside canonical carrier identity. The optional proactive bytes and -dedicated payload responses are untrusted transport until the sole verifier checks them against the -canonical application header and transaction commitment. The actor receives `DataAvailable` only -after Core has materialized the concrete block and its existing availability predicate succeeds; -neither carrier delivery nor a raw payload response may bypass that application DA gate. +The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified +Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not +payload reconstruction or the runtime transition from delivered acknowledgments to that input. + +Transaction bytes remain outside header RBC. The existing Reed-Solomon dissemination, +acknowledgment, reconstruction, and transaction-commitment checks remain responsible for data +availability. ## 2. Model and notation -The current prototype assumes: +Version one assumes: - one static, ordered, stake-weighted committee for a run; - Byzantine stake strictly below one third of total stake; @@ -112,36 +133,22 @@ The current prototype assumes: - no committee reconfiguration; and - no state retirement until a safe recovery watermark is proved. -For one target slot, let `W` be total committee stake and `a` the target author's stake. The -implementation derives all weighted thresholds with checked integer arithmetic: +For total committee stake `W`, use the repository's integer thresholds: ```text -F = floor((W - 1) / 3) maximum Byzantine stake under faults < W/3 -V = F + 1 READY validity/amplification threshold -Q = W - F READY certification and carrier quorum threshold - -U = W - a stake outside the target author -b = F - a residual Byzantine non-author stake, when a <= F -M = floor(U / 2) + 1 strict non-author majority -C = floor((U + b) / 2) + 1 convergence threshold -O = M + b authoritative optimistic ECHO threshold +Q = floor(2W / 3) + 1 +V = floor(W / 3) + 1 ``` -`M`, `C`, and `O` apply when `a <= F`, so the target author may be Byzantine. ECHO, VOTE, and ACK -stake exclude that author. If `a > F`, the global fault bound makes the target author honest; the -receiver-specific authenticator on exact content is then an immediate authoritative-delivery -predicate, and the locally fixed high-stake author seeds READY for the independent certificate -path. READY always counts the whole committee. For equal stake and `n = 3f + 1`, `Q = 2f + 1` and -`V = f + 1`. +For equal stake and `n = 3f + 1`, these are `Q = 2f + 1` and `V = f + 1`. Two independent round numbers are used: - `carrier_round` belongs to the fast physical DAG and advances from authenticated admission; -- `consensus_round` belongs to the authoritative logical Starfish projection. +- `consensus_round` belongs to the certified logical Starfish projection. There is no fixed mapping between them. Carrier rounds may run ahead while a consensus vertex is -waiting for authoritative carrier delivery, application data availability, a leader decision, or -eligible strong parents. +waiting for RBC delivery, data availability, a leader decision, or certified strong parents. ## 3. One physical DAG, two logical projections @@ -153,26 +160,24 @@ The same stored objects have two disjoint interpretations: 1. **Optimistic carrier projection.** Authenticated carriers and their weak parent references pace carrier creation and transport RBC statements. This projection is allowed to differ temporarily between honest validators. -2. **Authoritative consensus projection.** Authoritatively delivered and data-available consensus - vertices, immutable strong parents, and exact delivery frontiers drive Starfish voting, - certification, skip/commit decisions, and linearization. The `O` fast-delivery rule may admit a - vertex before its independent `Q`-READY certificate; honest validators eventually agree on the - safe projection. +2. **Certified consensus projection.** Eligible consensus vertices, immutable strong parents, and + certified delivery frontiers drive Starfish voting, certification, skip/commit decisions, and + linearization. Honest validators eventually agree on this projection. Weak carrier edges never become strong/order edges, even if their targets later deliver. A node must not construct its own filtered consensus parent set from an optimistic carrier after the fact: that would make the authenticated content have different consensus meaning at different nodes. This separation prevents a Byzantine carrier from poisoning an honest carrier. A quorum-sized weak -parent set can contain selectively disseminated or invented Byzantine references. Waiting for all -such references to become authoritatively delivered would make the honest child permanently unusable. +parent set can contain up to `f` selectively disseminated or invented Byzantine references. Waiting +for all such references to become RBC-delivered would make the honest child permanently unusable. Weak edges are therefore permanently nonblocking and nonordering. Only the explicitly encoded -strong parents and the authoritative exact frontier constrain consensus. +strong parents and certified frontier constrain consensus. ## 4. Canonical objects -The canonical codec implements the following logical types. Field widths, enum codes, maximum -lengths, and golden bytes are part of the implemented runtime contract. +The milestone-two codec implements the following logical types. Field widths, enum codes, maximum +lengths, and golden bytes are frozen before runtime integration. ```rust struct CarrierHeaderV1 { @@ -184,7 +189,6 @@ struct CarrierHeaderV1 { weak_parents: Vec, transactions_commitment: TransactionsCommitment, - application_header: Option, data_acknowledgments: Vec, phase_batch: Vec, consensus_vertex: Option, @@ -194,8 +198,6 @@ struct CarrierHeaderV1 { enum RbcPhaseStatementV1 { Echo { target: BlockReference }, Ready { target: BlockReference }, - Vote { target: BlockReference }, - Ack { target: BlockReference }, } struct ConsensusVertexV1 { @@ -217,10 +219,8 @@ enum LeaderChoiceV1 { ``` The author of an embedded phase statement or consensus vertex is the author of its enclosing -carrier. An outer authenticator therefore authenticates the ordered batch without a separate tag -or signature per statement. `application_header = None` is a control-only V1 carrier; -`Some(exact_header)` selects the V2 identity/wire grammar and binds the complete canonical -application header into the carrier digest. +carrier. An outer authenticator therefore authenticates the whole batch without a separate tag or +signature per statement. For non-genesis carrier round `r`: @@ -241,16 +241,17 @@ virtual. Every embedded consensus vertex has a positive consensus round and an e choice. These conventions avoid making genesis a second, partially authenticated wire format. Weak references are syntax and pacing declarations, not availability assertions. Their target -headers need not be present to authenticate, admit, process, or authoritatively deliver the -enclosing carrier. +headers need not be present to authenticate, admit, process, or RBC-deliver the enclosing carrier. Acknowledgments have one canonical logical order: first the unique maximal suffix shared with `[own_prev] || weak_parents`, then all remaining acknowledgments in their original relative order. The content digest commits to this expanded, suffix-first sequence, while the wire codec stores the shared suffix as an intersection index and retains the order-significant extras. Non-canonical wire -aliases and duplicate acknowledgments are rejected. The current standalone runtime emits this -field empty. In particular, a carrier acknowledgment is not an application-DA oracle and cannot -bypass the concrete Core gate described below. +aliases and duplicate acknowledgments are rejected. An honest author creates an acknowledgment +only after the exact target is locally RBC-delivered and its transaction data reconstructs to the +committed root. The acknowledgment becomes usable as data-availability evidence only after its +enclosing carrier is also locally RBC-delivered; an optimistically admitted Byzantine carrier +cannot create inconsistent availability facts at different validators. `delivery_frontier` has exactly one indexed entry per committee authority. `None` denotes that authority's fixed genesis/empty prefix. A `Some(reference)` entry must name the same authority as @@ -275,24 +276,24 @@ phase batch and optional consensus vertex. It excludes: - recovery or transport metadata. The byte grammar uses a one-byte format version, fixed field markers, big-endian fixed-width -integers, and explicit vector lengths. It does not add a mutable string domain to the block -identity. Control-only carriers retain the frozen V1 content/wire versions (`01`/`81`); a carrier -with an exact application header uses V2 (`02`/`82`). The canonical identity codec is handwritten; +integers, and explicit vector lengths. It does not add a `starfish:block-ref:v2` string to the block +identity. The format byte and unambiguous grammar distinguish this carrier layout; changing the +layout requires a new version and new golden vectors. The canonical identity codec is handwritten; serde or bincode framing is never hashed. -`Ref` is `author:u16 || carrier_round:u32 || digest:[u8;32]`; every integer is big-endian and every -vector count is `u16`. The expanded identity grammar is: +Milestone two freezes the version-one identity grammar as follows. `Ref` is +`author:u16 || carrier_round:u32 || digest:[u8;32]`; every integer is big-endian and every vector +count is `u16`. ```text -00 version // 01 control-only, 02 with application header +00 01 01 author:u16 02 carrier_round:u32 03 own_prev:Ref 04 weak_count:u16 weak:Ref[] 05 transactions_commitment:[u8;32] -0A canonical_application_header // present exactly for version 02 06 acknowledgment_count:u16 expanded_acknowledgments:Ref[] -07 phase_count:u16 (phase:u8 target:Ref)[] // ECHO=0, READY=1, VOTE=2, ACK=3 +07 phase_count:u16 (phase:u8 target:Ref)[] // ECHO=0, READY=1 08 consensus_present:u8 [ConsensusVertexV1] 09 creation_time_ns:u64 ``` @@ -303,13 +304,13 @@ frontier entries use `0=None` and `1=Some(Ref)`; leader choices use `1=Vote` and reserved for virtual genesis and is rejected on the wire). The canonical transport codec replaces the expanded acknowledgment field with `intersection_start:u16 || extra_count:u16 || extras`, where the intersection is the unique maximal suffix of `[own_prev] || weak_parents`. Decoding expands and -recompresses this field and rejects aliases. The compressed transport versions set the high bit: -`81` for control-only V1 and `82` for application-bearing V2. +recompresses this field and rejects aliases. To keep the two byte grammars self-describing, this +compressed transport form starts with `00 81`; only expanded identity content starts with `00 01`. -The codec caps canonical carrier content at 4 MiB, weak and strong parents at the committee size, +Version one caps canonical carrier content at 4 MiB, weak and strong parents at the committee size, the frontier at exactly the committee size when projected, and encoded phase batches at -`min(6n, 2048)`. Six entries per authority leave bounded spillover above the four-phase steady-state -arrival rate; the scheduler limitations are described in Section 8.3. +`min(4n, 2048)`. The `4n` bound gives two times the expected `2n` steady-state phase arrival rate; +the scheduler still needs the fair-prefix and active-window rules described in Section 8.3. Consensus vertices are referenced by their exact enclosing `BlockReference` plus their declared `consensus_round`. Because there is at most one consensus vertex per carrier, that pair identifies @@ -337,7 +338,7 @@ In MAC mode, author `A` computes entry `q` for recipient `Q_q` over a fixed-widt binds at least: ```text -STARFISH_RBC_DAG_V2 +STARFISH_RBC_DAG_V1 carrier-authentication kind and scheme protocol_instance committee_id @@ -347,9 +348,9 @@ carrier_round canonical carrier content digest ``` -The full vector accompanies every normally disseminated MAC carrier in the current prototype, -including relayed carriers. A receiver verifies only the entry at its own committee index. It -neither verifies nor vouches for the remaining entries. +The full vector accompanies every normally disseminated MAC carrier in version one, including +relayed carriers. A receiver verifies only the entry at its own committee index. It neither verifies +nor vouches for the remaining entries. A carrier received directly from its author and the same carrier received through a relay are both authentication-eligible when the local entry verifies. This is receiver-specific transferable @@ -359,14 +360,13 @@ signature and provides no non-repudiation. The vector is deliberately not an RBC value and has no consistency invariant. A Byzantine author may attach different vectors to the same content reference, including a vector with a valid tag for one recipient and garbage for another. Correctness therefore depends only on the local entry and on -the embedded four-phase protocol, never on agreement about the vector bytes. +the embedded Bracha protocol, never on agreement about the vector bytes. Each node persists one exact vector variant with its carrier for restart and relay. The preference order is locally generated, directly author-received, then first relayed variant with a valid local -entry. The implementation does not merge unverified entries from different vectors. Exact carrier -recovery after phase evidence may return canonical content without a vector; that recovery can -unblock phase progress, authoritative delivery, and READY certification, but it does not create -authenticated carrier admission or fast-clock stake. +entry. Version one does not merge unverified entries from different vectors. Header recovery after +authenticated quorum phase evidence may return canonical content without a vector; that recovery +can unblock RBC delivery but does not create optimistic carrier admission. Public-signature modes use the same context-bound carrier statement without a recipient field and the same embedded RBC/consensus logic. They exist for controlled performance comparison, not as @@ -386,23 +386,14 @@ Authenticated CarrierAdmitted Candidate && Authenticated && accepted by the carrier admission window -AuthoritativeDelivered - exact content is locally fixed, authenticated from a necessarily honest author, supported by - O author-excluding ECHO stake, or certified by Q READY - -ReadyCertified - Q READY stake names the exact retained content; recorded independently even if fast delivery - happened earlier - -DataAvailable - control-only carrier, or Core has materialized the exact application block and verified its - committed payload +Delivered + local Bracha instance reached Q READY and pinned matching canonical content PrefixClosed - AuthoritativeDelivered && DataAvailable && exact own_prev prefix is closed + Delivered && DataAvailable && exact own_prev prefix is closed VertexProjected (orthogonal to the carrier lifecycle) - this carrier's optional consensus vertex is eligible in the authoritative projection + this carrier's optional consensus vertex is eligible in the certified projection Included a committed Starfish anchor frontier names this carrier in its deterministic delta @@ -411,11 +402,9 @@ Ordered the complete included delta is available and the carrier has been deterministically output ``` -`Candidate` alone permits bounded staging and exact-reference recovery. `CarrierAdmitted` permits -immediate phase-batch processing and fast-pacemaker counting. `AuthoritativeDelivered` permits -phase replay even at a node whose author MAC entry was poisoned. `ReadyCertified` is useful audit -and fallback evidence but is not a second output gate after a valid fast delivery. -`PrefixClosed` permits frontier inclusion. +`Candidate` alone permits bounded staging and digest-based recovery. `CarrierAdmitted` permits +immediate phase-batch processing and fast-pacemaker counting. `Delivered` permits phase replay even +at a node whose author MAC entry was poisoned. `PrefixClosed` permits frontier inclusion. `VertexProjected` alone permits the optional vertex to supply Starfish vote/certifier/leader evidence. It is not a later state of every carrier: a carrier with no eligible optional vertex may still become prefix-closed, included by another anchor's frontier, and ordered. @@ -427,17 +416,17 @@ unauthenticated candidate advance the fast clock. | Consumer | Required local authority | |---|---| | Header retention/recovery | `Candidate` | -| Process embedded RBC statements | `CarrierAdmitted`, or `AuthoritativeDelivered` for replay | +| Process embedded RBC statements | `CarrierAdmitted`, or `Delivered` for replay | | Fast carrier clock | `CarrierAdmitted` | -| Exact carrier recovery | requested `Candidate` whose bytes recompute the target reference | -| Accept application payload | carrier-authorized header plus commitment-verified bytes | -| Application data-availability fact | concrete, data-available Core block only | +| Transaction/shard synchronization | `Candidate` | +| Count a data-availability acknowledgment | `Delivered` enclosing carrier | | Delivery frontier | `PrefixClosed` target carrier | | Starfish QC, skip, and anchor commit | `VertexProjected` consensus vertex | -| Application payload output | cumulative `Included` delta, with every member authoritatively delivered and application-data-available | +| Application payload output | `Included` delta, with every member delivered/data-available | -Fast-pacemaker counting is exactly `CarrierAdmitted`. Authoritative delivery is intentionally not a -second pacing input because it would change the sequential per-author slot accounting. +The initial implementation should keep fast-pacemaker counting exactly at `CarrierAdmitted`; using +`Delivered` as an additional stronger pacing input can be added only if the executable model shows +that it cannot change the sequential slot accounting. ## 7. Fast carrier pacemaker @@ -452,28 +441,23 @@ author, but each author contributes stake once. A validator advances from carrie 1. its own carrier at round `r` has been fixed and persisted; and 2. it has admitted distinct-author carrier stake `Q` at round `r`. -The executable prototype admits authenticated carriers at most two rounds ahead of the local open -round and retains canonical unsolicited carrier content up to 64 rounds ahead. Authenticated -retained carriers may later be promoted sequentially; candidate-only content retains no admission -authority. -Farther unsolicited values are discarded by the benchmark resource guard. Buffered carriers never -skip missing local rounds. These values are engineering bounds, not a proof-safe asynchronous -retirement rule. The next local carrier records the selected quorum as -`{ own_prev } union weak_parents`; a missing weak-parent body never blocks later action. +Future carriers are bounded and buffered; they do not skip missing local rounds. The next local +carrier records the selected quorum as `{ own_prev } union weak_parents`. A missing weak-parent body +never blocks the next carrier or any later consensus action. This clock replaces the current `starfish-rbc` rule that requires a quorum of RBC-clean previous round headers before proposal. It does not make admitted carriers consensus votes. Leader/vote/skip waiting conditions move to the independent consensus projection and cannot block the creation of a -carrier needed to transport ECHO, VOTE, ACK, or READY. +carrier needed to transport ECHO or READY. Honest validators emit empty control heartbeats when they have no application transactions. -Without heartbeats, low load can stop the four-phase waves and violate RBC liveness. Every carrier, +Without heartbeats, low load can stop the ECHO/READY waves and violate RBC liveness. Every carrier, including an empty heartbeat, is itself an RBC value and has an authenticator sidecar. The first prototype retains the current run's carrier/RBC state and rate-limits carrier creation. A production design needs a proved runahead and backpressure rule. A hard carrier/consensus skew cap must not suppress control heartbeats, because those heartbeats may be exactly what allows the -authoritative projection and committed frontier to catch up. +certified frontier to catch up. ## 8. Embedded all-carrier reliable broadcast @@ -484,106 +468,89 @@ RbcSlot = (protocol_instance, committee_id, carrier_author, carrier_round) RbcValue = BlockReference ``` -Receiver authentication is the admission capability for that exact value. A non-author that first -admits a value durably locks and queues ECHO. The target author never contributes ECHO, VOTE, or ACK -stake, and target-authored statements in those phases are ignored. Locally fixing a carrier is an -authoritative delivery predicate for its creator, but it does not manufacture author stake in an -author-excluding certificate. +Authenticating the carrier is INIT for that value. The local author records its own ECHO when it +atomically fixes and persists the carrier. An honest non-author that first admits a value queues one +ECHO for that slot. -For a potentially Byzantine target (`a <= F`), the reducer applies these weighted transitions: - -```text -admit exact content -> ECHO -ECHO stake >= M -> VOTE -ECHO stake >= C or VOTE stake >= C -> ACK -ACK stake >= C or READY stake >= V -> READY -ECHO stake >= O -> authoritative optimistic delivery -READY stake >= Q -> independent READY certification -``` +INIT is not silently counted as the author's ECHO at remote validators. The author's recorded local +ECHO is queued into a later carrier like every other phase action; it counts locally immediately and +remotely only after the enclosing carrier is admitted or delivered. This preserves the standard +quorum accounting without excluding the broadcaster's stake. -ECHO, VOTE, and ACK use distinct-author stake outside the target author; READY uses the full -committee. VOTE does not require local carrier admission, only the latched ECHO evidence and exact -retained target content. ACK may arise from `C` ECHO or `C` VOTE and does not require the node to -have sent VOTE. READY may arise from `C` ACK or the standard `V` READY amplification path. Local -phase evidence counts immediately after its durable lock, before later carrier dissemination. +Local phase actions are inserted into the next possible carrier's `phase_batch`. The enclosing +carrier's authentication makes its author the phase sender. For each target slot: -If `a > F`, the target author is necessarily honest. A valid receiver-bound authenticator on the -exact author value is therefore an immediate authoritative delivery predicate. The locally fixed -high-stake author also seeds READY; because its stake is at least `V`, the ordinary amplification -and `Q` certification path still completes independently. +- an authority emits at most one ECHO; +- an authority emits at most one READY; +- ECHO and READY choices may differ; +- `Q` ECHO stake creates a READY obligation; +- `V` READY stake creates a READY obligation; and +- `Q` READY stake plus pinned matching content locally delivers the exact carrier. -Reaching `O` is authoritative delivery, not a speculative hint. The reducer pins the exact carrier, -advances its delivered prefix when the DA and predecessor conditions hold, and may use its optional -vertex in the authoritative projection. `Q` READY records the slower certificate even if the same -value already delivered through the fast predicate. A certificate reached first also satisfies the -delivery predicate. No threshold over a bare digest can expose a phase or deliver a placeholder. +A READY obligation is not yet a local READY. If the target content is absent, the validator first +recovers it from authenticated ECHO/READY authors, validates the exact reference, and durably pins +it. Only then does it persist the slot-global READY lock, count its local READY, and enqueue the +statement. Consequently every honest READY author is a real content holder. A quorum trigger remains +latched while recovery is pending. -For every local slot, ECHO, VOTE, ACK, READY, delivery, and READY certification each lock at most -one target; different phases may safely name different targets. For every remote sender, the node -also records at most one target per phase and slot. Exact replay is idempotent, and a later -same-sender equivocation is ignored before it can add stake or allocate a second phase choice. +Local ECHO and READY count toward their thresholds before network dissemination. Evidence is +tracked per candidate, while local send and delivery locks are slot-global. Each remote authority +contributes stake at most once per phase and target slot; exact replay is idempotent and a later +equivocation is ignored before allocating another candidate. ### 8.1 Processing rule -After canonical validation and admission of the outer authenticator, a receiver processes the -phase batch in canonical encoded order immediately. An authenticated carrier beyond the two-round -admission window is retained but neither contributes pacemaker stake nor creates ECHO; its batch -gains authority only after sequential promotion or an independent authoritative-delivery predicate. -A retained candidate alone has no phase authority. Once authorized, processing does not wait for -the enclosing carrier's READY certificate, application DA, prefix closure, or projection. Waiting -would create a recursion because its controls are needed to deliver earlier carriers. +After canonical validation and outer authentication, a receiver processes the phase batch in its +canonical encoded order immediately. It must not wait for the enclosing carrier itself to be RBC +delivered, data-available, dependency-closed, or projected. Waiting would create a recursion: a +carrier's controls are required to deliver earlier carriers, while those earlier deliveries may be +required to create the next consensus vertex. If the local carrier authenticator is missing or invalid, its phase batch is not processed on -candidate receipt. If that exact outer carrier later becomes authoritatively delivered, the stored -batch is replayed under that delivery capability. Thus poisoned vector entries delay optimistic -admission but cannot permanently suppress controls selected by the four-phase protocol. Direct -phase messages are never an alternative authority source. +candidate receipt. If that exact outer carrier later becomes locally RBC-delivered, the stored batch +may be replayed under the local delivery capability. Thus poisoned vector entries delay optimistic +admission but cannot permanently suppress controls selected by RBC. Phase targets are strictly older carrier rounds, making a single carrier's replay acyclic. Local arrival order between different authenticated carriers is still observable and can affect which -Byzantine equivocation encounters a slot-global guard first. The ordered journal persists each -batch entry and its replay cursor; recovery must replay that order, never reconstruct choices by -sorting carriers after a restart. +Byzantine equivocation encounters a slot-global guard first. Recovery must replay the persisted +ingress journal, never reconstruct choices by sorting carriers after a restart. ### 8.2 Header recovery -An honest ECHO, VOTE, ACK, or READY author retains the exact target carrier content before exposing -that phase. A validator may therefore latch threshold evidence before it has the bytes, record the -union of phase senders as candidate holders, and request the target from those holders. Recovery -content is accepted only when canonical decoding recomputes the requested `BlockReference` and the -context/committee checks succeed. Retention precedes any new local phase lock. - -Recovery request/response is out-of-band byte transfer, not quorum testimony, admission, or a new -phase. A valid response can satisfy an already allocated evidence obligation but cannot create one. -The prototype retries recorded holders after GST. Its separate carrier catch-up mechanism requests -one exact `(author, round)` at a time and serves only retained locally authored outbound bytes; it -does not transfer ranges, certificates, checkpoints, committed observer history, or arbitrary late -state. +An honest ECHO or READY author must retain the target carrier content. A validator that observes a +threshold before receiving the target requests it from several recorded phase authors. Recovery +content is accepted only when canonical validation recomputes the requested reference. + +Recovery request/response remains an out-of-band data-transfer optimization in the first +prototype. It is not quorum testimony and does not change the on-DAG phase transcript. A `Q` ECHO +set contains honest holders, and a `V` READY set contains at least one honest holder, so retrying +authenticated holders eventually obtains the value after GST. ### 8.3 Batching and fairness Phase batches are bounded. The encoded order is preserved and processed as an authenticated log; two different orders intentionally identify different carriers. A deterministic fair queue must prevent Byzantine traffic for one slot -from starving honest ECHO/VOTE/ACK/READY actions for other slots. In steady state, one authority can -owe up to four actions for each of `n` previous carriers. The executable model retains an unbounded -pending FIFO and drains the first `6n` statements eligible for the carrier being built (capped at +from starving honest ECHO/READY actions for other slots. In steady state, one authority can owe one +ECHO and one READY for each of `n` previous-round carriers, so `2n` is the expected arrival rate and +not a safe capacity. The executable model retains an unbounded pending FIFO and drains the first +`4n` statements eligible for the carrier being built (capped by the version-one codec limit of 2,048 statements). A temporarily ineligible future-round statement remains in its stable queue position but does not block older eligible work behind it. This exercises backlog, runahead, and batching without pretending to solve adversarial fairness. A bounded runtime must use a fair -per-slot scheduler, reserve spillover above the four-phase arrival rate, and enforce an +per-slot scheduler, reserve strictly more than `2n` statements per carrier, and enforce an active-slot window so delayed work drains instead of remaining at permanent saturation. -## 9. Authoritative consensus vertices +## 9. Certified consensus vertices A carrier contains zero or one `ConsensusVertexV1`. The carrier remains valid and pace-eligible if -the optional vertex is malformed relative to authoritative projection state; only the optional -vertex is excluded from the consensus projection. +the optional vertex is malformed relative to local certified state; only the optional vertex is +excluded from the consensus projection. A consensus vertex authored by `A` at consensus round `c > 0` is eligible only when: -1. its enclosing carrier is authoritatively delivered by a local-fixed, honest-author, - `O`-ECHO, or `Q`-READY predicate; +1. its enclosing carrier is locally RBC-delivered; 2. its enclosing carrier's transaction data is available and it closes `A`'s carrier prefix as defined in Section 10; 3. its strong parents name distinct-author eligible consensus vertices at exactly `c - 1` whose @@ -609,8 +576,7 @@ strong parents block only this optional vertex. They never block the enclosing c batch, the fast clock, or later honest RBC progress. The consensus pacemaker preserves Starfish's separate advance and creation conditions, evaluated -only over eligible consensus vertices. A `Q`-READY certificate is not additionally required after -an authoritative optimistic delivery: +only over eligible consensus vertices: - **A1:** advance from `c - 1` to `c` after eligible distinct-author stake `Q` at `c - 1`; - **A2:** do not advance until the local consensus vertex at `c - 1` has been fixed; @@ -648,8 +614,7 @@ choice—must resolve Byzantine conflicts. ## 10. Closed delivery prefixes and frontiers -Authoritative carrier delivery alone is not application data availability and is not a compact -availability proof for a Byzantine author's later carrier. A +RBC delivery alone is not a compact availability proof for a Byzantine author's later carrier. A Byzantine author may deliver round `r` with an `own_prev` that names an unavailable fork at `r - 1`. Therefore a frontier component is a contiguous exact prefix, not simply the highest delivered round. @@ -657,9 +622,8 @@ delivered round. For authority `A`, begin at its fixed genesis/empty prefix. A carrier `(A, r, R)` extends the local closed prefix only when: -- `R` is authoritatively delivered; -- for an application carrier, Core has materialized the exact application block and its committed - transaction data satisfies the existing Starfish availability predicate; +- `R` is locally RBC-delivered; +- its transaction data satisfies the existing Starfish availability predicate; - `r` is exactly one more than the current prefix round; and - `R.own_prev` equals the exact current prefix tip. @@ -676,24 +640,23 @@ ensures that committed frontiers never regress or switch Byzantine forks. The containing carrier cannot name itself in its encoded frontier. For an eligible consensus vertex, its declared author component must equal its carrier's `own_prev` prefix tip. Once the -enclosing carrier is authoritatively delivered and data-available, its **effective frontier** -replaces that one component with the enclosing carrier. This makes a committed anchor's own -application payload eligible without waiting for a later anchor while preserving exact prefix -continuity. +enclosing carrier is delivered and data-available, its **effective frontier** replaces that one +component with the enclosing carrier. This makes a committed anchor's own application payload +eligible without waiting for a later anchor while preserving exact prefix continuity. The liveness target is deliberately precise: -> Every honest carrier that authoritatively delivers and becomes data-available eventually appears in a +> Every honest carrier that RBC-delivers and becomes data-available eventually appears in a > committed effective-frontier delta. No guarantee is made for a malformed or permanently off-prefix Byzantine carrier. Guaranteeing all -authoritatively delivered Byzantine forks would require an antichain or sparse exception structure -rather than one compact prefix tip per authority. +RBC-delivered Byzantine forks would require an antichain or sparse exception structure rather than +one compact prefix tip per authority. ## 11. Starfish certification, commit, and skip Starfish's logical leader schedule and commit rules run over eligible consensus vertices only. -Carrier admission, weak parents, phase targets, candidate headers, and merely retained carriers +Carrier admission, weak parents, phase targets, candidate headers, and merely delivered carriers cannot act as voters, certifiers, leaders, non-votes, or reachability evidence. For a scheduled leader slot at consensus round `c`, every eligible voter publishes one immutable @@ -721,52 +684,37 @@ view is never a no-vote. `NoVote` is explicit, authenticated, immutable, and slo An honest validator persists its leader-choice lock before exposing the carrier that contains it; it cannot emit `NoVote` and later vote for a late leader in the same logical voting slot. -Decision planning follows the existing Starfish newest-to-oldest rule. For an undecided slot, the -deciding anchor is the first committed leader in the already-decided later sequence at least three -rounds ahead. Final skips are traversed, but an intervening undecided slot is a hard barrier; a -validator must not scan past it to whichever later direct certificate happens to be visible first. -Only the longest finalized prefix is published. Direct versus indirect evidence may be observed at -different times, but it cannot change the committed-leader sequence. - Skipping a Byzantine leader role discards only that optional consensus value. It does not discard the enclosing application carrier. If that carrier later becomes part of a closed prefix, a later committed frontier orders its payload. -Every consensus consumer enforces this type boundary: voter caches, leader support, potential -certificates, direct/indirect decisions, reachability, and the linearizer reject non-projected -carrier facts. Application data availability is not inferred from this projection or from carrier -acknowledgment references; it enters only through the typed Core materialization callback. +Every consensus consumer in the current Starfish committer must be audited for the new type +boundary: voter caches, leader support, potential certificates, direct/indirect decisions, +reachability, and the linearizer must reject non-projected carrier facts. Data-availability +acknowledgments are the deliberate exception: they become usable when their enclosing carrier is +RBC-delivered, which breaks a projection/availability circularity while still excluding merely +optimistic evidence. ## 12. Frontier-delta linearization -Let `A_k` be the `k`th committed leader in the finalized leader sequence, whether directly or -indirectly committed, `F_k` its effective frontier, `J_k` the cumulative joined committed frontier, -and `Closure(F)` the union of the exact per-author self-chain prefixes named by `F`. Maintain: +Let `F_k` be the effective frontier carried by committed anchor `A_k`, and let `Closure(F_k)` be the +union of the exact per-author self-chain prefixes named by `F_k`. Maintain: ```text -J_0 = [None; committee_size] -J_k = componentwise_exact_join(J_(k-1), F_k) -Delta = Closure(J_k) \ Closure(J_(k-1)) +C_0 = fixed genesis carriers +C_k = C_(k-1) union Closure(F_k) +Delta = C_k \ C_(k-1) ``` -The join compares exact self-chain lineage, not round numbers. It accumulates advances from -concurrent committed anchors and prevents a later partial frontier from erasing or regressing a -component already committed. Before outputting `Delta`, a validator requires every exact carrier -to be authoritatively delivered and every application member to pass the concrete Core DA gate. -Exact carrier recovery and the dedicated verified-payload path supply missing material. - -Frontiers are applied strictly in increasing finalized-leader order. A later leader used to decide -an older slot is planning evidence only until every older slot has resolved; its frontier is not -applied ahead of an indirectly committed older leader. Thus different projection arrival orders -may classify a leader as direct or indirect at different times, but produce the same ordered anchor -and frontier-delta sequence. +Before outputting `Delta`, a validator waits until every exact member is locally RBC-delivered and +data-available. RBC totality and erasure-coded recovery supply missing content for honest committed +frontiers. All validators deterministically order the same delta by `(carrier_round, author, content_digest)`. Because a closed author prefix advances by exactly one carrier round, this key already preserves mandatory `own_prev` order. -Weak parents, strong consensus edges, optional-vertex projection time, -ECHO/VOTE/ACK/READY target references, +Weak parents, strong consensus edges, optional-vertex projection time, ECHO/READY target references, recovery provenance, and MAC-vector variants never constrain application payload ordering. Strong edges order consensus decisions and dominate frontiers, but a late-projecting optional vertex must not retroactively add an edge between payloads already output. This fixed ordering also ensures that @@ -778,18 +726,15 @@ design removes. In an all-honest synchronous interval, batching can realize this conceptual schedule: ```text -t = 0 carrier k authenticates an exact application header; non-authors lock ECHO -t = delta ECHO stake reaches O: authoritative delivery; VOTE/ACK obligations are queued -t = 2delta ACK stake reaches C: READY is queued -t = 3delta READY stake reaches Q: the independent certificate is recorded +t = 0 carrier k contains a new application header (RBC INIT) +t = delta carrier k+1 contains ECHOs for k +t = 2delta carrier k+2 contains READYs for k +t = 3delta carrier k is RBC-delivered; a later carrier may project new consensus work ``` -The `O` fast path deliberately makes authoritative delivery available after the ECHO wave instead -of waiting for the READY certificate. The VOTE/ACK/READY path remains necessary for convergence, -totality, and independent certification under adverse schedules. For `a > F`, receiver-authenticated -exact content takes the honest-author fast branch even before the ECHO threshold. Application -output still waits for DA, logical consensus, and the cumulative committed frontier, so the RBC -delivery schedule is not itself a transaction-latency claim. +The embedded design does not make Bracha RBC require fewer communication delays than the direct +baseline. Its performance hypothesis is that carrier batching reduces frames, scheduling work, and +duplicated control metadata while the fast carrier clock overlaps certification with dissemination. Implementation ordering is latency-critical. On carrier ingress, authenticate, apply its phase batch, execute newly enabled delivery/prefix/projection transitions, and only then decide what the @@ -808,8 +753,8 @@ An authoritative implementation must persist proof-critical choices before expos 1. journal typed authenticated inbound provenance, exact bytes, and its local ingress sequence; 2. before fixing a local slot, construct and persist the typed candidate plus its exact canonical carrier bytes and reference; -3. persist local ECHO, VOTE, ACK, READY, delivery-promise, READY-certificate, explicit - leader-choice, carrier-slot, and consensus-slot locks that match retained exact content; +3. persist local ECHO, READY, explicit leader-choice, delivery, carrier-slot, and consensus-slot + locks that match that retained candidate (recovered content is likewise retained before READY); 4. persist the exact authentication sidecar and an outbound-exposure marker, and only then send the carrier; and 5. after restart, replay the journal in recorded order and retransmit the identical carrier and @@ -822,45 +767,35 @@ until every lock encoded by that carrier is durable. Every persisted slot, candidate, lifecycle predicate, journal entry, and outbound-carrier key is namespaced by both `protocol_instance` and `committee_id`; storage from another run or committee -cannot satisfy a local lock or quorum. The current authoritative storage path is additionally -separated as autonomous WAL V4, and raw journal records start with `SRD5`. V4 prevents traces from -the older planning-only promise semantics from being reinterpreted as authoritative fast delivery; -`SRD5` prevents older record layouts from decoding as the current ECHO/VOTE/ACK/READY journal. +cannot satisfy a local lock or quorum. Hash-sorting recovered carriers is not a valid reconstruction rule. Byzantine equivocation can make arrival order determine which value a local slot-global guard selects, and a different restart order could make one honest authority appear to send conflicting phases. -The Core store has a bounded latest-frontier receipt shape and can atomically persist an application -commit with that receipt, but that alone is not a complete actor-to-Core recovery handshake. Until -startup re-emits exactly the actor-WAL frontier suffix newer than Core's durable cursor, applies it -before the `Ready`/application-production barrier, and rejects a cursor the actor cannot reconcile, -a crash between actor durability and Core application remains a fail-stop boundary. Actor replay -must not be described as exactly-once output without that composed contract. - The proof model retains all proof-critical carrier, phase, prefix, and consensus state for the run. -The executable runtime admits at lookahead `2` and bounds newly arriving unsolicited content at -lookahead `64` solely to keep a faulty or descheduled peer from growing the benchmark process -without limit. This is not a protocol-safe retirement rule: an honest carrier may be delayed more -than 64 rounds under asynchrony. Recovery of an exact already-requested value is exempt. Before -authoritative garbage collection is enabled, the design needs a common certified or committed -retirement watermark that preserves: - -- pending four-phase totality and exact-content recovery; +The milestone-three shadow bounds newly arriving unsolicited content to a fixed recent-round +window solely to keep a faulty peer from growing an observational benchmark process without limit. +This is not a protocol-safe retirement rule: an honest INIT may be delayed longer than that under +asynchrony. Recovery of an exact already-requested value is exempt. Before authoritative garbage +collection is enabled, the design needs a common certified or committed retirement watermark that +preserves: + +- pending Bracha totality and header recovery; - exact self-prefix expansion from the last committed frontier; - committed-anchor reconstruction for a late validator; and - deterministic replay of local locks. Resource bounds still required before authoritative deployment include a proof-safe future and -retirement window, per-peer candidate caps, a fair bounded phase backlog, bounded peer/network -bridges, a rate-limited control heartbeat, a bounded payload runahead policy, and checkpointed -disk-backed recovery. The actor's primary ingress is bounded, but not every bridge, per-peer outbox, -or retained reducer collection is yet bounded end to end. Resource exhaustion is excluded from the -initial proof model and must be measured in the prototype. Any run in which work is shed is invalid -for direct/shadow comparison; +retirement window, per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, +a bounded payload runahead policy, and checkpointed disk-backed recovery. Shadow input and output +channels are bounded and shed observational work instead of backpressuring direct consensus, but +the reference reducer's retained history and per-transition validation are not yet bounded-runtime +architecture. Resource exhaustion is excluded from the initial proof model and must be measured in +the prototype. Any run in which work is shed is invalid for direct/shadow comparison; `starfish_rbc_dag_shadow_comparison_valid` must remain `1` for the entire measured interval. A live pipeline does not have equal cumulative direct and shadow delivery counters at an arbitrary -instant: the embedded four-phase protocol normally leaves a short shadow tail. Benchmark verification therefore +instant: embedded ECHO/READY normally leaves a short shadow tail. Benchmark verification therefore requires monotone nonzero direct, shadow, and paired-match progress, no conflict outcome, and bounds both the current unpaired slots (`<= 4n` per validator) and the oldest unpaired round lag against the newest current-process observation (`<= 4`). These are empirical benchmark coverage guards, not @@ -875,10 +810,8 @@ at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slo exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, healthy proactive rounds receive a repair grace period, and exact synchronization is rate-limited -per peer. Exact synchronization transfers only one requested `(author, round)` and only from the -author's retained local outbound map. It has no range response, certified checkpoint, observer -history, or bounded-suffix state-transfer protocol; a sufficiently late or fresh node cannot be -reconstructed by this mechanism alone. +per peer. Requested historical slots remain recoverable beyond the benchmark-only unsolicited +retention window. Autonomous benchmark validity is separate from delivery comparison validity. `starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and local-carrier counters @@ -895,42 +828,35 @@ The design is not complete until at least the following claims are proved or fal 1. **Receiver-authentication integrity.** An honest receiver admits a carrier attributed to an honest author only if that author created the public proof or the receiver's MAC entry. A MAC is not public non-repudiation, and a Byzantine endpoint knows its own pairwise key. -2. **Four-phase agreement and integrity.** Target-author exclusion, the `M/C/O/V/Q` intersections, - slot-global ECHO/VOTE/ACK/READY locks, per-sender phase locks, and exact value binding prevent - conflicting authoritative deliveries at honest validators. -3. **Fast-predicate safety.** `a > F` really implies an honest author, and an `O = M + b` ECHO set - contains enough honest non-author support to make the value unique and force the fallback to - converge on it. -4. **RBC totality and certification.** If one honest validator authoritatively delivers a value, - VOTE/ACK convergence, READY amplification, heartbeats, and exact holder recovery cause every - honest validator to deliver it and eventually record the same `Q`-READY certificate. -5. **Admission isolation.** Mere authentication/admission can change fast pacing and phase replay, - but cannot alter a QC, leader decision, skip, commit, or output. Only a proved authoritative - delivery predicate plus prefix/DA/projection gates crosses that boundary. -6. **Weak-edge non-poisoning.** A missing or equivocating weak parent cannot block delivery, +2. **RBC agreement and integrity.** Slot-global ECHO/READY locks, quorum intersection, and exact + value binding prevent two conflicting carrier values from being delivered by honest validators. +3. **RBC totality.** If one honest validator delivers a value, heartbeats, READY amplification, and + holder recovery cause every honest validator eventually to deliver the same value. +4. **Optimistic isolation.** Carrier admission can change only fast pacing and RBC processing; it + cannot alter a QC, leader decision, skip, commit, acknowledgment certificate, or output order. +5. **Weak-edge non-poisoning.** A missing or equivocating weak parent cannot block delivery, projection of unrelated honest vertices, or application ordering. -7. **Consensus-slot uniqueness.** Honest validators create/vote once per +6. **Consensus-slot uniqueness.** Honest validators create/vote once per `(author, consensus_round)`, and Byzantine conflicts cannot both acquire honest quorum support. -8. **Prefix and join comparability.** Every accepted frontier component is an exact extension of - its strong ancestors, and every cumulative committed join is monotone on exact lineage. -9. **Projection safety.** Erasing weak edges and optional consensus metadata that is not +7. **Prefix comparability.** Every accepted frontier component is an exact extension of its strong + ancestors and of every earlier committed component. +8. **Projection safety.** Erasing weak edges and optional consensus metadata that is not `VertexProjected` leaves a valid execution of the Starfish commit/skip rules over immutable strong edges; it does not erase otherwise orderable carrier payloads. -10. **Deterministic ordering.** Equal committed-anchor histories imply equal cumulative frontier - joins, closures, deltas, and transaction order at all honest validators. -11. **Data availability.** No application enters an output delta until Core contains the concrete - block and its committed transaction root has been reconstructed and verified. +9. **Deterministic ordering.** Equal committed anchors imply equal frontier closures, deltas, and + transaction order at all honest validators. +10. **Data availability.** No carrier enters an output delta until its committed transaction root + can be reconstructed and verified. ## 16. Liveness obligations Under partial synchrony and fair processing, the design must establish: -1. Honest authors of aggregate stake at least `Q` continually create authenticated carriers after - GST, so the sequential fast clock advances without Byzantine participation. -2. Empty heartbeat carriers drain every honest ECHO/VOTE/ACK/READY backlog even when application load is +1. `Q` honest authors continually create authenticated carriers after GST, so the sequential fast + clock advances without Byzantine participation. +2. Empty heartbeat carriers drain every honest ECHO/READY backlog even when application load is zero. -3. Every honest carrier is authoritatively delivered and eventually `Q`-READY certified at every - honest validator. +3. Every honest carrier is RBC-delivered at every honest validator. 4. Existing Starfish data availability eventually closes every honest author's exact carrier prefix. 5. Honest consensus vertices with quorum strong parents continue to appear despite arbitrary @@ -938,58 +864,46 @@ Under partial synchrony and fair processing, the design must establish: 6. The projected Starfish pacemaker eventually commits infinitely many honest anchors. 7. Honest frontier construction is fair: every newly closed honest carrier prefix is eventually included in a committed frontier. -8. Waiting for a committed delta cannot block forever for honest applications because every named - exact carrier is already authoritatively delivered and the concrete application DA condition is - part of frontier eligibility. +8. Waiting for a committed delta cannot block forever because every named exact carrier is already + RBC-delivered and data-available by frontier eligibility. The guaranteed payload-liveness statement covers every honest on-prefix carrier. Selectively disseminated, malformed, or off-prefix Byzantine carriers may be ignored. ## 17. Required executable tests -The executable model and composed runtime tests should cover at minimum: +Milestone two begins with an isolated deterministic model, not production network wiring. At +minimum it must cover: - `n = 4, f = 1` and `n = 7, f = 2` all-honest progress; -- split Byzantine author values and receiver-selective poisoned vector entries; +- split Byzantine INIT values and receiver-selective poisoned vector entries; - valid relayed local MAC entries and invalid vector variants; -- weighted and unequal-stake `F/a/U/b/M/C/O/V/Q` threshold goldens, including `a > F`; -- ECHO/VOTE/ACK/READY local locks, per-sender replay/equivocation, ordered batch replay, and restart; -- `M` ECHO to VOTE, `C` ECHO-or-VOTE to ACK, `C` ACK to READY, `O` authoritative delivery, and - independent `Q`-READY certification; -- evidence-before-content recovery from phase holders, including VOTE/ACK without local admission; +- ECHO/READY equivocation, replay, reordering, and evidence-before-header recovery; - zero application load with heartbeat-only RBC completion; -- two-round admission, 64-round authenticated retention, and future carriers that cannot jump the - local sequential clock; +- future carriers that cannot jump the local sequential clock; - `f` permanently missing weak parents without blocking honest carrier or consensus progress; -- an authoritatively delivered Byzantine carrier above an unavailable self-chain gap; +- a delivered Byzantine carrier above an unavailable self-chain gap; - conflicting Byzantine consensus vertices in one logical slot; - explicit vote/no-vote conflicts and direct plus indirect commit/skip; -- different projection arrival orders where a later direct certificate precedes an earlier - indirect anchor, with byte-identical committed-leader and cumulative frontier-delta sequences; - frontier fork, regression, and strong-parent dominance rejection; -- concurrent committed anchors producing the same cumulative joined frontier and byte-identical - output deltas without regression; -- delayed concrete Core data availability followed by eventual prefix inclusion, with raw payload - receipt unable to bypass the gate; +- equal committed anchors producing byte-identical output deltas; +- delayed data availability followed by eventual prefix inclusion; - crash points before and after each persisted lock and outbound-carrier write; and -- crash points before and after Core frontier application, including atomic receipt/commit, - exact-replay idempotence, conflicting/stale cursor rejection, suffix replay before `Ready`, and - explicit failure of the buffered-WAL crash-safety case; and -- persisted actor restart with byte-identical local retransmission, poisoned-tag candidate - retention, exact recovery, and no phase/output exposure before its matching durable locks; and +- persisted shadow-actor restart with byte-identical retransmission against an identical recovered + direct-header history, bounded overload, poisoned-tag candidate retention, exact recovery, and + paired delivery observations against the current direct RBC kernel. Full validator restart is + excluded until the authoritative direct-RBC locks are durable; and - autonomous actor progress at `n = 4` and `n = 7`, no steady-state repair polling on healthy proactive rounds, exact-slot synchronization with idempotent late responses and per-peer rate limiting, multi-round convergence after a validator falls behind, control-only WAL reopen, - distinct authentication namespace, V4/`SRD5` stale-trace rejection, and bounded exact-slot sync - that does not pretend to be checkpoint transfer; and -- composed frontier-authority runs in which every exact application header is authoritatively + distinct authentication namespace, and an integration check that direct Starfish-RBC continues + committing while the observational carrier clock advances; and +- composed frontier-authority runs in which every exact application header is embedded-RBC delivered, all honest nodes release the same deterministic application order without duplicates, - the direct INIT/phase/batch/pull paths and legacy committer have no authority, and the - frontier/application/WAL progress gates remain valid. + the legacy committer is disabled, and the frontier/application/WAL progress gates remain valid. Property tests should mutate every canonical field and verify carrier-reference binding, while -golden tests freeze the V1 control and V2 application encodings, append-only phase codes, and flat -vector length. +golden tests freeze the version-one encoding and flat vector length. ## 18. Complexity and benchmark plan @@ -1002,7 +916,7 @@ The first fair benchmark matrix includes: - Sailfish++ as a certified signature-free comparison. Hold committee, load, transaction size, topology, latency injection, dissemination fanout, duration, -timeouts, and build constant. Report carrier, vector, ECHO, VOTE, ACK, READY, recovery, payload, +timeouts, and build constant. Report carrier/INIT, vector, ECHO, READY, recovery, transaction/shard, and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained state. @@ -1010,20 +924,20 @@ state. Batching can reduce the number of separately scheduled RBC control messages, but it does not remove their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout -transport is added. Comparison mode sends both direct and embedded transcripts; standalone mode -does not. The default crash-safe reference profile applies preflighted transitions in place but -fsyncs every accepted transition and still performs synchronous reducer/storage work, so it is a -correctness/replay instrument rather than a fair latency profile. The explicit buffered-WAL profile -keeps the exact framed event path but syncs only on clean shutdown and therefore cannot support -crash-safety claims. Benchmark output reports appended and durable WAL work separately. +transport is added. Shadow mode also sends both direct and embedded transcripts. The default +crash-safe reference profile fsyncs each accepted transition and validates through a clone-based +reducer; it is a correctness/replay instrument, not a protocol-performance result. The explicit +buffered-WAL profile keeps the exact framed event path but syncs only on clean shutdown and therefore +cannot be used for crash-safety claims. Benchmark output reports appended and durable WAL work +separately. The clone-based reducer remains intentionally unoptimized until measurement shows it +matters. A matched 10-validator local sequence on 2026-08-11 used a full 60-second active transaction window, the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and the buffered benchmark WAL. Milestone-five idle carriers use the same resolved 600 ms Push leader timeout as Starfish-RBC; application and encodable phase carriers are immediate. The harness waits through generator warmup, snapshots cumulative counters at the active boundary, and drains final latency -samples. The table and milestone narrative below predate the current four-phase V4 authority model -and remain historical evidence rather than a current-protocol result. +samples. | Profile | Verdict | TPS | Block latency | E2E latency | Outbound | |---|---:|---:|---:|---:|---:| @@ -1067,36 +981,50 @@ toward the roughly 600 ms unsafe Starfish-MAC reference without weakening RBC or ## 19. Contained implementation milestones -The historical milestones produced the current bounded prototype: - -1. **Canonical carrier plane (implemented):** V1 control and V2 application carrier identities, - full-vector sidecars, two independent clocks, ordered phase batches, exact recovery, and - deterministic codec/model/journal tests. -2. **Weighted optimistic RBC (implemented):** author-excluding ECHO/VOTE/ACK thresholds, READY - fallback/certification, the high-stake honest-author branch, `O` authoritative delivery, and - durable slot-global locks for every phase. -3. **Standalone authority boundary (implemented, opt-in):** carriers and the dedicated verified - application-payload path are the only ingress authority. The direct INIT/phase/header service, - generic batch/pull path, and legacy application committer are absent or rejected. -4. **Logical consensus and output (implemented):** independently numbered vertices, quorum strong - parents, explicit Vote/NoVote choices, exact prefixes, authoritative projection, Starfish - commit/skip, cumulative joined committed frontiers, and deterministic application deltas. -5. **Durable actor replay (implemented within the documented scope):** V4/`SRD5` WAL replay restores - retained content, phase/delivery/consensus locks, locally authored outbound bytes, and projection - state. This is not a claim of full validator restart or general late-node state transfer. -6. **Remaining production work:** complete the end-to-end proof; bound every queue and retained - collection; add proof-safe checkpoints, state transfer, retirement, and graceful shutdown; then - compare latency with direct `starfish-rbc`, unsafe `starfish-mac`, signature variants, and - Sailfish++ before attempting tree dissemination. +Every milestone is committed separately. + +1. **Protocol specification (this document):** lock the two clocks, lifecycle, full-vector sidecar, + embedded Bracha transitions, certified prefix/frontier, commit/skip boundary, proof obligations, + and experiment plan. No protocol code or CLI selector is added. +2. **Canonical codec and executable model (implemented):** isolated carrier, phase, consensus, + frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable + journal models; and deterministic adversarial simulations. No network or existing consensus path + changes. +3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the + current direct `starfish-rbc` service, cache the validated committee/domain identity rather than + re-hashing all public keys per carrier, journal ingress and local locks, and compare embedded + versus direct RBC delivery through current-process paired observations. Direct RBC remains + authoritative; shadow results never affect proposals or commits. The reference WAL/reducer is a + correctness instrument, not yet an interpretable protocol-performance path. +4. **Optimistic carrier clock (implemented, opt-in control shadow):** run a separately namespaced, + control-only heartbeat carrier plane with the distinct authenticated-admission latch, sequential + quorum clock, bounded future buffer, exact-slot synchronization, durable restart, and clock + validity metrics while consensus still uses the current direct baseline. +5. **Authoritative embedded RBC (implemented, opt-in):** encode exact canonical application headers + in version-two carriers, durably reconcile their origins, schedule application/phase carriers + immediately, and remove direct ECHO/READY/delivery authority. Direct INIT remains payload + transport; composed tests assert zero direct ECHO/READY traffic and positive embedded delivery. +6. **Certified consensus projection (implemented):** add optional independently numbered consensus + vertices, quorum strong parents, explicit timeout-bound leader choices, contiguous exact + delivery frontiers, durable slot/choice locks, and a live clean-only direct committer. Malformed + optional vertices do not poison their enclosing carrier. +7. **Frontier linearizer and recovery (implemented within the actor's fail-stop scope):** commit + deterministic frontier deltas, reconstruct prefixes, decisions, and anchors from the ordered + WAL, disable the legacy application committer, and output exact application references once. + Full-validator crash recovery and proof-safe late-node state transfer remain deferred because + the direct payload-transport baseline does not yet persist its own proof-critical RBC state. +8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, + signature Starfish variants, and Sailfish++ before attempting tree dissemination. +9. **Tree dissemination:** distribute vector sub-bundles with redundant routing and a direct timeout + fallback; do not change RBC or consensus semantics. ## 20. Decisions intentionally deferred -The following production choices remain unresolved and must be proved or measured: +The following values are not safe to guess in the documentation milestone and must be resolved by +the executable model or measured prototype: -- production maximum future-carrier buffer and payload runahead (the executable prototype keeps - admission lookahead `2`, retains at most `64` future rounds for temporarily descheduled peers, - and discards farther unsolicited carriers before admission/retention; these are benchmark resource - parameters rather than protocol safety constants); +- production maximum future-carrier buffer and payload runahead (the executable model deliberately + uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); - whether the shared Starfish leader-timeout policy needs a separately proved adaptive low-load rule; the prototype intentionally does not introduce a second heartbeat timeout; - a safe state-retirement, garbage-collection, and late-catch-up watermark; @@ -1104,7 +1032,6 @@ The following production choices remain unresolved and must be proved or measure - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and - the tree topology, redundancy, and fallback timers. -Mixed direct/standalone or incompatible carrier/WAL deployments must be rejected. The current code -has fail-closed protocol/storage namespaces but no feature handshake, so homogeneous configuration -is an operational precondition. A dedicated `starfish-rbc-dag` selector should be added only with -explicit capability/version negotiation. +Mixed `starfish-rbc`, `starfish-rbc-dag`, and version-one/version-two deployments must be rejected +by protocol-instance negotiation. The provisional `starfish-rbc-dag` selector is added only after +the codec/model milestone establishes a distinct stable version. From 7e5cc23b86f7e5c75308c7f7aa934f8cab389108 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:40:03 +0200 Subject: [PATCH 54/62] Revert "Make RBC-DAG frontiers authoritative" This reverts commit 65dd57e38eee19da664087703a0d0474e2c79d76. --- README.md | 35 +- crates/orchestrator/src/measurements.rs | 116 ++--- crates/starfish-core/src/block_handler.rs | 77 +--- crates/starfish-core/src/core.rs | 118 +---- .../starfish-core/src/core_thread/spawned.rs | 34 -- crates/starfish-core/src/metrics.rs | 77 +--- crates/starfish-core/src/net_sync.rs | 30 +- .../src/starfish_rbc_dag/journal.rs | 18 - .../src/starfish_rbc_dag/model.rs | 40 +- .../src/starfish_rbc_dag/projection.rs | 50 +-- .../src/starfish_rbc_dag_shadow.rs | 424 ++++++------------ .../src/starfish_rbc_dag_shadow_service.rs | 165 ++----- crates/starfish-core/src/syncer.rs | 53 +-- crates/starfish-core/src/validator.rs | 59 +-- docs/starfish-rbc-dag-protocol.md | 51 +-- 15 files changed, 286 insertions(+), 1061 deletions(-) diff --git a/README.md b/README.md index 855c189c..ea0b261c 100644 --- a/README.md +++ b/README.md @@ -57,16 +57,14 @@ the comparison shadow with `--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact application headers in version-two carriers and make embedded ECHO/READY/delivery their sole certification authority. Direct INIT remains payload transport, but direct ECHO/READY cannot clean -blocks in that mode. Committed projected anchors now release deterministic carrier-frontier deltas, -and those deltas are the sole application ordering/output authority; the legacy Starfish committer -is disabled. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for +blocks in that mode. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for Starfish-RBC by default); application and encodable phase carriers are emitted immediately. Autonomous carriers now embed durably locked consensus vertices with quorum strong parents, explicit Vote/NoVote choices, and exact delivery frontiers. Only RBC-delivered, data-available, -prefix-closed vertices enter the projection or its leader decisions. Frontier output retains exact -application references and is rebuilt from the ordered WAL on actor reopen. Full validator crash -recovery and proof-safe late-node state transfer remain outside this milestone. Shadow traffic shares the +prefix-closed vertices enter the projection or its leader decisions. The existing Starfish DAG is +still the temporary application-output scaffold; the committed frontier linearizer is the next +milestone. Shadow traffic shares the validator's network socket and bandwidth, and deployment requires a homogeneous new-binary committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; `--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean @@ -78,10 +76,10 @@ observational path was disabled or shed work and the comparison must be discarde production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between the cumulative direct and shadow delivery counters. Autonomous runs instead require -`starfish_rbc_dag_shadow_clock_valid == 1`, local-carrier/WAL progress, advancing carrier rounds, -in-window local carrier, embedded-RBC delivery, projected-vertex, clean projected-commit, and -committed-frontier application progress, plus bounded clock-state gauges. The current queue budget -supports at most 60 validators in mirror mode and 20 in autonomous mode. +`starfish_rbc_dag_shadow_clock_valid == 1`, heartbeat/WAL progress, advancing carrier rounds, +in-window embedded-RBC delivery, projected-vertex and clean projected-commit progress, and bounded +clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 +in autonomous mode. A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared @@ -93,30 +91,21 @@ nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Sta | Autonomous comparison, direct RBC authoritative | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | | Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | -| Frontier output authority (milestone seven) | VALID 10/10 | 950.25 | 2,020.9 ms | 2,082.6 ms | 0.70 MB/s | The milestone-five run produced 10,722 embedded application deliveries, reached carrier rounds 458–459, and ended with zero pending recovery. It also proves that the earlier 250 ms experimental heartbeat was not the latency cause: application and phase carriers are already event-driven, and using the shared 600 ms timeout did not restore the direct baseline. The remaining slowdown is an expected warning about the transitional architecture—the old direct DAG still serializes proposal -creation on embedded RBC cleanliness. Milestone six lets the optimistic carrier clock advance -independently and feeds only certified vertices into the logical committer; its result motivated -milestone seven's removal of the remaining legacy output gate. +creation on embedded RBC cleanliness. Milestone six now lets the optimistic carrier clock advance +independently and feeds only certified vertices into the logical committer; milestone seven must +remove the remaining legacy output gate by committing deterministic frontier deltas. The milestone-six run reached carrier rounds 356–359 with 35,506 carrier deliveries, 8,059 application deliveries, 8,487 projected vertices, 830 clean direct commits, and zero pending recovery. Its further latency increase is a structural red flag, not a projection-speed claim: certified decisions currently run alongside the old clean-predecessor/output path, so the benchmark -still pays for both. Milestone seven's measurement below is the first one after removing that +still pays for both. The next measurement is meaningful only after milestone seven removes that legacy gate. -Milestone seven disables the legacy committer in embedded-authority mode, advances application -production from the optimistic carrier clock, and releases exact application headers only through -committed frontier deltas. Its run reached carrier round 795 on every validator, delivered 79,035 -application carriers, released 77,870 applications through 1,880 committed frontiers, and ended -with zero pending recovery. This recovers 60.4% of milestone six's block-latency regression and -75.9% of its E2E regression, but 2.02/2.08 seconds is still well above the roughly 600 ms unsafe -Starfish-MAC target. The next performance work must measure and shorten the certified-projection -round/commit pipeline rather than reintroducing legacy certification or ordering. The local harness starts its timer after transaction-generator warmup, subtracts warmup counters, and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 51af511c..600aab66 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -695,38 +695,6 @@ impl MeasurementsCollection { .is_some_and(|(last, first)| last > first) } - fn count_bucket_sum_increased( - &self, - label: &str, - scraper_id: ScraperId, - buckets: &[&str], - ) -> bool { - let Some(series) = self.active_window_series(label, scraper_id) else { - return false; - }; - let bucket_is_monotonic = buckets.iter().all(|bucket| { - series.windows(2).all(|window| { - window[1].count_buckets.get(*bucket).copied().unwrap_or(0) - >= window[0].count_buckets.get(*bucket).copied().unwrap_or(0) - }) - }); - let totals = series - .iter() - .map(|measurement| { - buckets.iter().fold(0usize, |total, bucket| { - total.saturating_add( - measurement.count_buckets.get(*bucket).copied().unwrap_or(0), - ) - }) - }) - .collect::>(); - bucket_is_monotonic - && totals - .last() - .zip(totals.first()) - .is_some_and(|(last, first)| last > first) - } - fn count_bucket_is_always_zero( &self, label: &str, @@ -1018,10 +986,6 @@ impl MeasurementsCollection { .parameters .node_parameters .starfish_rbc_dag_autonomous_clock; - let embedded_rbc_authority = self - .parameters - .node_parameters - .starfish_rbc_dag_embedded_rbc_authority; let shadow_comparison_enabled = shadow_enabled && !shadow_autonomous_clock_enabled; let shadow_valid_scrapers = self .data @@ -1181,75 +1145,60 @@ impl MeasurementsCollection { .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); let autonomous_buffered_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let every_autonomous_scraper_has_progress = shadow_autonomous_clock_valid_scrapers - .iter() - .all(|scraper_id| { - self.count_bucket_sum_increased( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - &["heartbeat,accepted", "application_carrier,accepted"], - ) && self.count_bucket_increased( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - "delivery,shadow", - ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_appended_batches_total", - *scraper_id, - ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_appended_records_total", - *scraper_id, - ) && self.scalar_counter_increased( - "starfish_rbc_dag_projected_vertices_total", - *scraper_id, - ) && self.count_bucket_increased( - "starfish_rbc_dag_projection_decisions_total", - *scraper_id, - "direct_commit", - ) && (!embedded_rbc_authority - || self.count_bucket_increased( + let every_autonomous_scraper_has_progress = + shadow_autonomous_clock_valid_scrapers + .iter() + .all(|scraper_id| { + self.count_bucket_increased( "starfish_rbc_dag_shadow_inputs_total", *scraper_id, - "frontier,committed", + "heartbeat,accepted", ) && self.count_bucket_increased( "starfish_rbc_dag_shadow_inputs_total", *scraper_id, - "frontier,application", - )) - && self.scalar_gauge_increased( + "delivery,shadow", + ) && self.scalar_counter_increased( + "starfish_rbc_dag_shadow_wal_appended_batches_total", + *scraper_id, + ) && self.scalar_counter_increased( + "starfish_rbc_dag_shadow_wal_appended_records_total", + *scraper_id, + ) && self.scalar_counter_increased( + "starfish_rbc_dag_projected_vertices_total", + *scraper_id, + ) && self.count_bucket_increased( + "starfish_rbc_dag_projection_decisions_total", + *scraper_id, + "direct_commit", + ) && self.scalar_gauge_increased( "starfish_rbc_dag_shadow_carrier_round", *scraper_id, - ) - && self.latest_scalar_greater_than( + ) && self.latest_scalar_greater_than( "starfish_rbc_dag_shadow_carrier_round", *scraper_id, 1.0, - ) - && self.latest_scalar_equals( + ) && self.latest_scalar_equals( "starfish_rbc_dag_shadow_pending_recovery", *scraper_id, 0.0, - ) - && self.gauge_always_at_most( + ) && self.gauge_always_at_most( "starfish_rbc_dag_shadow_phase_backlog", *scraper_id, autonomous_phase_backlog_bound as f64, - ) - && self.gauge_always_at_most( + ) && self.gauge_always_at_most( "starfish_rbc_dag_shadow_admitted_authors", *scraper_id, self.parameters.nodes as f64, - ) - && self.gauge_always_at_most( + ) && self.gauge_always_at_most( "starfish_rbc_dag_shadow_admitted_stake", *scraper_id, f64::MAX, - ) - && self.gauge_always_at_most( + ) && self.gauge_always_at_most( "starfish_rbc_dag_shadow_buffered_authenticated", *scraper_id, autonomous_buffered_bound as f64, ) - }); + }); let shadow_autonomous_clock_valid = shadow_autonomous_clock_enabled && expected_shadow_nodes != 0 && shadow_autonomous_clock_valid_nodes == expected_shadow_nodes @@ -1652,15 +1601,10 @@ mod test { Measurement { timestamp, count_buckets: HashMap::from([ - ( - "application_carrier,accepted".to_owned(), - wal_durable_records, - ), + ("heartbeat,accepted".to_owned(), wal_durable_records), ("delivery,shadow".to_owned(), wal_durable_records), - ("frontier,committed".to_owned(), wal_durable_records), - ("frontier,application".to_owned(), wal_durable_records), ]), - count: wal_durable_records.saturating_mul(4), + count: wal_durable_records.saturating_mul(2), ..Measurement::default() }, ); diff --git a/crates/starfish-core/src/block_handler.rs b/crates/starfish-core/src/block_handler.rs index 76bc4269..001fa0a0 100644 --- a/crates/starfish-core/src/block_handler.rs +++ b/crates/starfish-core/src/block_handler.rs @@ -261,23 +261,30 @@ impl RealCommitHandler { let pending = committed.into_iter().skip(ready_count).collect(); (resulted_committed, pending) } +} - fn record_commit_metadata<'a>( +impl CommitObserver for RealCommitHandler { + fn handle_commit( &mut self, - committed: impl IntoIterator, - ) { + dag_state: &DagState, + committed_leaders: Vec<(Data, Option)>, + ) -> Vec { + let mut committed = self + .commit_interpreter + .handle_commit(dag_state, committed_leaders); let current_timestamp = runtime::timestamp_utc(); let metrics_active = self .metrics .metrics_active .load(std::sync::atomic::Ordering::Relaxed); - for commit in committed { - self.committed_leaders.push(commit.anchor); + for commit in &committed { + self.committed_leaders.push(commit.0.anchor); self.committed_count += 1; + // Chain rolling commit digest: hash(prev_digest || anchor.digest) let mut hasher = blake3::Hasher::new(); hasher.update(&self.commit_digest); - hasher.update(commit.anchor.digest.as_ref()); + hasher.update(commit.0.anchor.digest.as_ref()); self.commit_digest = *hasher.finalize().as_bytes(); let commit_index = self.committed_count; @@ -289,12 +296,13 @@ impl RealCommitHandler { self.metrics.commit_digest.set(digest_short as i64); } - for block in &commit.blocks { - let gap = commit.anchor.round.saturating_sub(block.round()); + for block in &commit.0.blocks { + let gap = commit.0.anchor.round.saturating_sub(block.round()); self.metrics.commit_gap.observe(gap as f64); let block_creation_time = block.meta_creation_time(); let block_latency = current_timestamp.saturating_sub(block_creation_time); + if block_creation_time.is_zero() || block_latency.as_secs() > 60 { tracing::debug!( "Latency of block {} is too large, \ @@ -317,9 +325,16 @@ impl RealCommitHandler { .committed_blocks .with_label_values(&[&block.authority().to_string()]) .inc(); + tracing::debug!("Latency of block {} is computed", block.reference()); } + // Tick the validator's `benchmark_duration` Prometheus counter, + // but only inside the active window. The orchestrator uses this + // counter as the timestamp on every scrape, which then becomes + // the denominator of the TPS rate. Anchor it to + // `active_start_micros` so warmup and wind-down seconds do not + // dilute the steady-state rate. if metrics_active { let active_start_micros = self .metrics @@ -334,19 +349,6 @@ impl RealCommitHandler { } } } - } -} - -impl CommitObserver for RealCommitHandler { - fn handle_commit( - &mut self, - dag_state: &DagState, - committed_leaders: Vec<(Data, Option)>, - ) -> Vec { - let mut committed = self - .commit_interpreter - .handle_commit(dag_state, committed_leaders); - self.record_commit_metadata(committed.iter().map(|commit| &commit.0)); if dag_state.consensus_protocol == ConsensusProtocol::StarfishBls { let mut pending = dag_state.read_pending_not_certified(); pending.append(&mut committed); @@ -368,39 +370,6 @@ impl CommitObserver for RealCommitHandler { resulted_committed } - fn handle_rbc_dag_commit( - &mut self, - dag_state: &DagState, - anchor: BlockReference, - applications: &[BlockReference], - ) -> Vec { - let blocks = applications - .iter() - .map(|reference| { - let block = dag_state - .get_storage_block(*reference) - .unwrap_or_else(|| panic!("committed RBC-DAG application {reference} missing")); - assert!( - dag_state.is_data_available(reference), - "committed RBC-DAG application {reference} is unavailable" - ); - block - }) - .collect::>(); - let commit = CommittedSubDag::new(anchor, blocks); - self.record_commit_metadata(std::iter::once(&commit)); - for block in &commit.blocks { - if block.round() > 0 { - self.transaction_observer(block.clone()); - } - } - self.sequenced_commit_count += 1; - self.metrics - .commit_availability_gap - .set((self.committed_count - self.sequenced_commit_count) as i64); - vec![commit] - } - fn recover_committed( &mut self, committed: AHashSet, diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index 68bf47ce..a6434175 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -75,10 +75,6 @@ pub struct Core { recovered_committed_leaders_count: Option, committer: UniversalCommitter, pub(crate) encoder: Encoder, - /// M7 application-production mode: direct Starfish headers are payload - /// descriptors only. Their dirty/clean DAG is no longer a consensus or - /// output authority, so raw threshold-clock progress may produce them. - rbc_dag_application_production: bool, } #[derive(Debug, Clone)] @@ -207,7 +203,6 @@ impl Core { recovered_committed_leaders_count: Some(committed_leaders_count), committer, encoder, - rbc_dag_application_production: false, }; if !unprocessed_blocks.is_empty() { @@ -508,11 +503,7 @@ impl Core { .utilization_timer .utilization_timer("Core::try_new_block"); - let proposal_round = if self.rbc_dag_application_production { - self.dag_state.threshold_clock_round() - } else { - self.dag_state.proposal_round() - }; + let proposal_round = self.dag_state.proposal_round(); tracing::debug!( "Attempt to construct block in round {} (proposal round {}). Current pending: {:?}", clock_round, @@ -529,9 +520,7 @@ impl Core { let protocol = self.dag_state.consensus_protocol; // Dual-DAG protocols: require clean parent quorum before creating a block. - if !self.rbc_dag_application_production - && protocol.uses_dual_dag() - && !protocol.is_starfish_rbc_single_dag() + if protocol.uses_dual_dag() && !protocol.is_starfish_rbc_single_dag() && clock_round > 1 && !self.dag_state.clean_parent_quorum(clock_round - 1) { @@ -548,9 +537,7 @@ impl Core { // Starfish-RBC that local header is dirty until the local RBC instance // delivers it; another clean quorum must not let us smuggle this dirty // mandatory parent into a proposal. - if !self.rbc_dag_application_production - && protocol.is_starfish_rbc() - && !protocol.is_starfish_rbc_single_dag() + if protocol.is_starfish_rbc() && !protocol.is_starfish_rbc_single_dag() && clock_round > 1 && self .last_own_block @@ -629,8 +616,7 @@ impl Core { clock_round ); } - if !self.rbc_dag_application_production - && protocol.uses_dual_dag() + if protocol.uses_dual_dag() && clock_round > 1 && block_references.is_empty() && !allows_minimal_refs @@ -812,7 +798,6 @@ impl Core { .dag_state .consensus_protocol .is_starfish_rbc_single_dag() - && !self.rbc_dag_application_production { pending_refs.into_iter().partition(|reference| { reference.round == 0 || self.dag_state.has_clean_vertex(reference) @@ -833,7 +818,6 @@ impl Core { .dag_state .consensus_protocol .is_starfish_rbc_single_dag() - && !self.rbc_dag_application_production { let before = block_references.clone(); block_references.retain(|r| r.round == 0 || self.dag_state.has_clean_vertex(r)); @@ -858,7 +842,6 @@ impl Core { let is_compressed_non_leader = self.dag_state.consensus_protocol.uses_compressed_refs() && self.committee.elect_leader(block_round) != self.authority; if self.dag_state.consensus_protocol.uses_dual_dag() - && !self.rbc_dag_application_production && block_round > 1 && !is_compressed_non_leader { @@ -1556,9 +1539,6 @@ impl Core { connected_authorities: &AHashSet, relaxed: bool, ) -> bool { - if self.rbc_dag_application_production { - return quorum_round > 0 && self.dag_state.threshold_clock_round() >= quorum_round; - } if quorum_round == 0 || self.dag_state.proposal_round() < quorum_round { return false; } @@ -1616,39 +1596,6 @@ impl Core { self.flush_pending_clean_refs(); } - /// Persist an M7 frontier delta without feeding the obsolete Starfish - /// clean-DAG commit/proposal watermarks back into block production. - pub fn handle_rbc_dag_committed_delta(&mut self, committed: Vec) { - let _timer = self - .metrics - .utilization_timer - .utilization_timer("Core::handle_rbc_dag_committed_delta"); - let mut commit_data = Vec::with_capacity(committed.len()); - for commit in &committed { - self.dag_state.update_last_committed_rounds(commit); - commit_data.push(CommitData::new( - commit, - self.dag_state.last_committed_rounds(), - )); - } - let store_start = std::time::Instant::now(); - self.store - .store_commits(commit_data) - .expect("Store RBC-DAG frontier commits should not fail"); - self.metrics - .store_commits_latency_us - .inc_by(store_start.elapsed().as_micros() as u64); - self.metrics.store_commits_count.inc(); - } - - pub(crate) fn enable_rbc_dag_application_production(&mut self) { - assert!( - self.dag_state.consensus_protocol.is_starfish_rbc(), - "RBC-DAG application production requires Starfish-RBC payload headers" - ); - self.rbc_dag_application_production = true; - } - pub fn write_commits(&mut self, _commits: &[CommitData]) {} pub fn take_recovered_committed(&mut self) -> (AHashSet, usize) { @@ -1967,63 +1914,6 @@ mod tests { })); } - #[test] - fn rbc_dag_application_production_does_not_wait_for_legacy_clean_delivery() { - let authority = 0; - let committee = Committee::new_for_benchmarks(4); - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new( - ®istry, - Some(committee.as_ref()), - Some("starfish-rbc"), - None, - ); - let dir = TempDir::new().unwrap(); - let recovered = DagState::open( - authority, - dir.path(), - metrics.clone(), - committee.clone(), - "honest".to_string(), - "starfish-rbc".to_string(), - &StorageBackend::Rocksdb, - false, - DisseminationMode::ProtocolDefault, - ); - let private_config = NodePrivateConfig::new_for_tests(authority); - let (mut core, _) = Core::open( - NoopBlockHandler, - authority, - committee.clone(), - private_config, - metrics, - recovered, - None, - ); - core.enable_rbc_dag_application_production(); - - let own_round_one = core - .try_new_block("new_blocks") - .expect("round-one application header should be creatable"); - let peers = [1, 2] - .into_iter() - .map(|peer| make_starfish_rbc_round_1_block(&committee, peer)) - .collect::>(); - core.add_headers(peers, DataSource::BlockBundleStreamingHeader); - - assert_eq!(core.dag_state().threshold_clock_round(), 2); - assert!(!core.dag_state().has_clean_vertex(own_round_one.reference())); - let round_two = core - .try_new_block("new_blocks") - .expect("RBC-DAG application production must follow the raw threshold clock"); - assert_eq!(round_two.round(), 2); - assert!( - round_two - .block_references() - .contains(own_round_one.reference()) - ); - } - #[test] fn mysticeti_bls_non_leader_can_build_round_2_with_prev_leader_parent() { let authority = 0; diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index b047b191..141b9a08 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -14,7 +14,6 @@ use crate::{ data::Data, metrics::{Metrics, UtilizationTimerExt}, starfish_rbc::PinnedRbcHeader, - starfish_rbc_dag_shadow::CommittedFrontierDeltaV1, syncer::{CommitObserver, Syncer, SyncerSignals}, types::{ AuthorityIndex, BlockReference, ProvableShard, ReconstructedTransactionData, RoundNumber, @@ -73,8 +72,6 @@ enum CoreThreadCommand { /// Apply locally delivered Starfish-RBC headers on the core thread. ApplyStarfishRbcDeliveries(Vec, oneshot::Sender<()>), ApplyStarfishRbcReference(crate::types::StarfishRbcReferenceV3, oneshot::Sender<()>), - /// Commit one deterministic clean carrier-frontier application delta. - ApplyStarfishRbcDagFrontier(CommittedFrontierDeltaV1, oneshot::Sender<()>), /// Store a Sailfish++ timeout certificate in DagState. ApplyTimeoutCert(SailfishTimeoutCert, oneshot::Sender<()>), /// Store a Sailfish++ no-vote certificate in DagState. @@ -226,19 +223,6 @@ impl Result { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::ApplyStarfishRbcDagFrontier( - delta, sender, - )) - .await; - receiver.await.expect("core thread is not expected to stop"); - } - /// Store a Sailfish++ timeout certificate on the core thread. pub async fn apply_timeout_cert(&self, cert: SailfishTimeoutCert) { let (sender, receiver) = oneshot::channel(); @@ -438,14 +422,6 @@ impl CoreThread { self.syncer.apply_starfish_rbc_reference(reference); sender.send(()).ok(); } - CoreThreadCommand::ApplyStarfishRbcDagFrontier(delta, sender) => { - metrics - .core_thread_tasks_total - .with_label_values(&["apply_starfish_rbc_dag_frontier"]) - .inc(); - self.syncer.apply_starfish_rbc_dag_frontier(delta); - sender.send(()).ok(); - } CoreThreadCommand::ApplyTimeoutCert(cert, sender) => { metrics .core_thread_tasks_total @@ -510,15 +486,6 @@ mod tests { Vec::new() } - fn handle_rbc_dag_commit( - &mut self, - _dag_state: &DagState, - _anchor: BlockReference, - _applications: &[BlockReference], - ) -> Vec { - Vec::new() - } - fn recover_committed( &mut self, _committed: AHashSet, @@ -566,7 +533,6 @@ mod tests { None, None, None, - false, ); CoreThreadDispatcher::start(syncer) } diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index aabc76f7..f99b0f4f 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -267,11 +267,9 @@ pub struct MetricReporter { /// and immediately before the measured local-benchmark interval begins. #[derive(Clone, Copy, Debug, Default)] pub struct AutonomousClockBenchmarkBaseline { - accepted_local_carriers: u64, + accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, - committed_frontiers: u64, - frontier_applications: u64, projected_vertices: u64, projection_decisions: u64, wal_batches: u64, @@ -299,8 +297,6 @@ struct AutonomousClockBenchmarkSummary { accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, - committed_frontiers: u64, - frontier_applications: u64, projected_vertices: u64, projection_decisions: u64, wal_batches: u64, @@ -345,13 +341,7 @@ fn summarize_autonomous_clock_benchmark( .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["heartbeat", "accepted"]) .get() - .saturating_add( - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["application_carrier", "accepted"]) - .get(), - ) - > baseline.accepted_local_carriers + > baseline.accepted_heartbeats && metrics .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) @@ -362,17 +352,7 @@ fn summarize_autonomous_clock_benchmark( .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get() - > baseline.delivered_applications - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get() - > baseline.committed_frontiers - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get() - > baseline.frontier_applications) + > baseline.delivered_applications) && metrics.starfish_rbc_dag_projected_vertices_total.get() > baseline.projected_vertices && metrics @@ -436,24 +416,6 @@ fn summarize_autonomous_clock_benchmark( .get() }) .sum(); - let committed_frontiers = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get() - }) - .sum(); - let frontier_applications = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get() - }) - .sum(); let projected_vertices = metrics .iter() .map(|metrics| metrics.starfish_rbc_dag_projected_vertices_total.get()) @@ -534,8 +496,6 @@ fn summarize_autonomous_clock_benchmark( accepted_heartbeats, delivered_carriers, delivered_applications, - committed_frontiers, - frontier_applications, projected_vertices, projection_decisions, wal_batches, @@ -566,15 +526,10 @@ pub struct VecHistogramReporter { impl Metrics { pub fn autonomous_clock_benchmark_baseline(&self) -> AutonomousClockBenchmarkBaseline { AutonomousClockBenchmarkBaseline { - accepted_local_carriers: self + accepted_heartbeats: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["heartbeat", "accepted"]) - .get() - .saturating_add( - self.starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["application_carrier", "accepted"]) - .get(), - ), + .get(), delivered_carriers: self .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) @@ -583,14 +538,6 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get(), - committed_frontiers: self - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get(), - frontier_applications: self - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get(), projected_vertices: self.starfish_rbc_dag_projected_vertices_total.get(), projection_decisions: self .starfish_rbc_dag_projection_decisions_total @@ -1812,12 +1759,10 @@ impl Metrics { table.add_row(row![ b->"Clock/WAL progress:", format!( - "heartbeats={}, carrier deliveries={}, application deliveries={}, committed frontiers={}, frontier applications={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", + "heartbeats={}, carrier deliveries={}, application deliveries={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, summary.delivered_carriers, summary.delivered_applications, - summary.committed_frontiers, - summary.frontier_applications, summary.projected_vertices, summary.projection_decisions, summary.wal_batches, @@ -2502,7 +2447,7 @@ mod tests { let metrics = &metrics[0]; metrics .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["application_carrier", "accepted"]) + .with_label_values(&["heartbeat", "accepted"]) .inc(); metrics .starfish_rbc_dag_shadow_inputs_total @@ -2535,14 +2480,6 @@ mod tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .inc(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .inc(); assert!( summarize_autonomous_clock_benchmark( &[Arc::clone(metrics)], diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index b9133379..5ce15418 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -1986,7 +1986,6 @@ impl NetworkSyncer } else { (None, None, None) }; - let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; let syncer = Syncer::new( core, NetworkSyncSignals { @@ -1999,7 +1998,6 @@ impl NetworkSyncer sf_msg_tx.clone(), starfish_rbc_service.clone(), starfish_rbc_dag_shadow_service.clone(), - embedded_rbc_authority, ); let initial_round = syncer.core().next_block_round(); let syncer = CoreThreadDispatcher::start(syncer); @@ -2223,6 +2221,7 @@ impl NetworkSyncer }) }); + let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; let embedded_rbc_committee_id = embedded_rbc_authority.then(|| { RbcCommitteeId::derive(&inner.committee) .expect("validated direct RBC committee must retain a stable identifier") @@ -2299,7 +2298,12 @@ impl NetworkSyncer embedded_rbc_committee_id .expect("embedded authority must cache its committee ID"), ) { - Ok(_) => {} + Ok(header) => { + event_inner + .syncer + .apply_starfish_rbc_deliveries(vec![header]) + .await; + } Err(error) => { shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); tracing::error!( @@ -2311,26 +2315,6 @@ impl NetworkSyncer } } } - ShadowServiceEventV1::FrontierCommitted(delta) => { - if embedded_rbc_authority { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .inc(); - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "carrier"]) - .inc_by(delta.carriers.len() as u64); - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .inc_by(delta.applications.len() as u64); - event_inner - .syncer - .apply_starfish_rbc_dag_frontier(delta) - .await; - } - } ShadowServiceEventV1::VertexProjected(reference) => { shadow_metrics .starfish_rbc_dag_projected_vertices_total diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index a68f2fe7..bdb4fb1d 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -1036,7 +1036,6 @@ pub struct WriteAheadJournalV1 { /// A transition checked against one exact journal prefix. Keeping only the /// newly appended events avoids cloning the complete durable history on every /// live shadow input. -#[cfg(test)] pub(crate) struct ValidatedJournalBatchV1 { base_event_count: usize, base_snapshot: Arc, @@ -1065,7 +1064,6 @@ impl WriteAheadJournalV1 { Ok(()) } - #[cfg(test)] pub(crate) fn validate_batch( &self, events: Vec, @@ -1082,7 +1080,6 @@ impl WriteAheadJournalV1 { }) } - #[cfg(test)] pub(crate) fn commit_validated_batch( &mut self, batch: ValidatedJournalBatchV1, @@ -1100,21 +1097,6 @@ impl WriteAheadJournalV1 { Ok(()) } - /// Apply a batch to the volatile authority snapshot without cloning its - /// complete retained history. This is safe only for a fail-stop caller - /// that persists the corresponding raw records before exposing any - /// effects and permanently poisons itself on a subsequent WAL failure. - pub(crate) fn apply_batch_unpublished( - &mut self, - events: Vec, - ) -> Result<(), JournalErrorV1> { - for event in events { - Arc::make_mut(&mut self.snapshot).apply(&event)?; - self.durable_events.push(event); - } - Ok(()) - } - pub fn record_authenticated_ingress( &mut self, authenticated: AuthenticatedCarrierV1, diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index 344eccb1..d4f0e64f 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -19,7 +19,7 @@ use std::{ use crate::{ committee::Committee, - crypto::{Blake3Hasher, TransactionsCommitment}, + crypto::Blake3Hasher, types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, }; @@ -277,14 +277,14 @@ struct CarrierRecord { } impl CarrierRecord { - fn new(carrier: CandidateCarrierV1, data_available: bool) -> Self { + fn new(carrier: CandidateCarrierV1) -> Self { Self { carrier, authenticated: false, admitted: false, phase_batch_cursor: 0, delivered: false, - data_available, + data_available: false, prefix_closed: false, } } @@ -343,7 +343,6 @@ pub struct RbcDagModel { committee_id: RbcDagCommitteeId, context: RbcDagContextV1, own_authority: AuthorityIndex, - intrinsic_empty_data_available: bool, revision: u64, lineage: ModelLineage, local_carrier_round: RoundNumber, @@ -384,7 +383,6 @@ impl RbcDagModel { committee_id, context, own_authority, - intrinsic_empty_data_available: false, revision: 0, lineage: [0; 32], local_carrier_round: 1, @@ -401,18 +399,6 @@ impl RbcDagModel { }) } - /// Enable the runtime rule that a carrier with the canonical empty - /// transaction commitment needs no external reconstruction oracle. The - /// generic M2 model leaves this disabled so tests can control DA - /// independently of carrier contents. - pub fn enable_intrinsic_empty_data_availability(&mut self) { - assert!( - self.carriers.is_empty(), - "intrinsic availability mode must be fixed before ingress" - ); - self.intrinsic_empty_data_available = true; - } - pub fn own_authority(&self) -> AuthorityIndex { self.own_authority } @@ -475,20 +461,6 @@ impl RbcDagModel { self.apply_input_traced(input).map(|log| log.effects()) } - /// Apply a transition before any of its effects are externally exposed. - /// The durable actor uses this fail-stop path to avoid cloning the entire - /// retained reducer history for every carrier. If persistence of the - /// returned trace fails, the caller must poison and terminate the actor; - /// it must never publish the returned effects. - pub(crate) fn apply_input_unpublished( - &mut self, - input: ModelInputRecord, - ) -> Result<(Vec, Vec), ModelError> { - let log = self.apply_input_traced(input)?; - let effects = log.effects(); - Ok((log.trace, effects)) - } - /// Deterministically reconstruct a model by replaying the original typed /// inputs in their recorded order. No round or lock is synthesized. pub fn replay_from_records( @@ -808,11 +780,9 @@ impl RbcDagModel { log: &mut TransitionLog, ) { let reference = carrier.reference(); - let intrinsic_data_available = self.intrinsic_empty_data_available - && carrier.header().transactions_commitment() == TransactionsCommitment::default(); self.carriers .entry(reference) - .or_insert_with(|| CarrierRecord::new(carrier, intrinsic_data_available)); + .or_insert_with(|| CarrierRecord::new(carrier)); // Canonical content can satisfy a previously latched recovery even if // the receiver-specific authenticator is invalid. @@ -2564,7 +2534,7 @@ mod tests { let reference = carrier.reference(); model .carriers - .insert(reference, CarrierRecord::new(carrier, false)); + .insert(reference, CarrierRecord::new(carrier)); let mut candidate_state = RbcCandidateState::default(); candidate_state.readies.extend([1, 2]); let mut slot = RbcSlotState::default(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 5c17e5ee..21797741 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -140,7 +140,6 @@ pub struct CertifiedProjectionModel { delivered_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, closed_prefixes: Vec>, vertices: BTreeMap, - vertices_by_round: BTreeMap>, consensus_slots: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, committed_frontier: DeliveryFrontierV1, committed_anchors: BTreeSet, @@ -166,7 +165,6 @@ impl CertifiedProjectionModel { delivered_slots: BTreeMap::new(), closed_prefixes: vec![Vec::new(); committee_size], vertices: BTreeMap::new(), - vertices_by_round: BTreeMap::new(), consensus_slots: BTreeMap::new(), committed_frontier: vec![None; committee_size], committed_anchors: BTreeSet::new(), @@ -273,24 +271,6 @@ impl CertifiedProjectionModel { .map(|projected| &projected.vertex) } - pub fn is_committed_anchor(&self, reference: ConsensusVertexReference) -> bool { - self.committed_anchors.contains(&reference) - } - - /// Whether the current committed prefix already contains every carrier - /// named by `frontier`. This is used when a later anchor first resolves an - /// older slot: an intermediate leader can subsequently be decided as a - /// logical commit even though its entire payload frontier was already - /// output by that later anchor. - pub fn committed_frontier_dominates(&self, frontier: &[Option]) -> bool { - frontier.len() == self.committed_frontier.len() - && frontier - .iter() - .copied() - .zip(self.committed_frontier.iter().copied()) - .all(|(older, committed)| self.is_exact_extension(older, committed)) - } - /// Current exact closed carrier-prefix frontier in authority order. pub fn closed_frontier(&self) -> DeliveryFrontierV1 { self.committee @@ -412,10 +392,6 @@ impl CertifiedProjectionModel { effective_frontier, }, ); - self.vertices_by_round - .entry(vertex_reference.consensus_round()) - .or_default() - .insert(vertex_reference); self.consensus_slots .entry((author, vertex_reference.consensus_round())) .or_default() @@ -706,15 +682,10 @@ impl CertifiedProjectionModel { &self, round: RoundNumber, ) -> impl Iterator { - self.vertices_by_round - .get(&round) - .into_iter() - .flatten() - .filter_map(|reference| { - self.vertices - .get(reference) - .map(|projected| (*reference, projected)) - }) + self.vertices + .iter() + .filter(move |(reference, _)| reference.consensus_round() == round) + .map(|(reference, projected)| (*reference, projected)) } fn voter_authors( @@ -832,10 +803,6 @@ impl CertifiedProjectionModel { effective_frontier: vec![None; self.committee.len()], }, ); - self.vertices_by_round - .entry(reference.consensus_round()) - .or_default() - .insert(reference); self.consensus_slots .entry((reference.author(), reference.consensus_round())) .or_default() @@ -1121,11 +1088,6 @@ mod tests { let anchor_carrier = clean(&mut model, anchor); let anchor_vertex = model.try_project(anchor_carrier).unwrap(); model.record_committed_anchor(anchor_vertex).unwrap(); - assert!( - model.committed_frontier_dominates( - &carriers.iter().copied().map(Some).collect::>() - ) - ); let regressing = second_round_candidate( &model, @@ -1142,10 +1104,6 @@ mod tests { model.record_committed_anchor(regressing_vertex), Err(CertifiedProjectionError::FrontierRegressesCommitted { authority: 0, .. }) )); - assert!( - !model - .committed_frontier_dominates(model.effective_frontier(regressing_vertex).unwrap()) - ); } #[test] diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 5caa8c7a..e12441e7 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -24,7 +24,10 @@ use crate::{ CarrierHeaderV1Args, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, carrier_genesis_reference, - journal::{IngressProvenanceV1, JournalErrorV1, JournalEventV1, WriteAheadJournalV1}, + journal::{ + IngressProvenanceV1, JournalErrorV1, JournalEventV1, ValidatedJournalBatchV1, + WriteAheadJournalV1, + }, model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, projection::{ CertifiedProjectionError, CertifiedProjectionModel, LeaderSlotV1, ProjectionDecisionV1, @@ -223,18 +226,6 @@ pub(crate) enum ShadowDeliveryComparisonV1 { }, } -/// Deterministic application output unlocked by one newly committed clean -/// projected anchor. Carrier references remain available for audit while the -/// application headers are already deduplicated and sorted by their carrier -/// position in the exact committed frontier delta. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct CommittedFrontierDeltaV1 { - pub(crate) anchor: ConsensusVertexReference, - pub(crate) frontier: Vec>, - pub(crate) carriers: Vec, - pub(crate) applications: Vec, -} - #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct ShadowOpenReportV1 { replayed_batches: u64, @@ -324,7 +315,8 @@ pub(crate) enum ShadowErrorV1 { }, MissingOutboundCandidate(BlockReference), MissingDeliveredCandidate(BlockReference), - PostModelJournal(JournalErrorV1), + PostDurabilityCommit(ModelError), + PostDurabilityJournal(JournalErrorV1), Projection(CertifiedProjectionError), Poisoned, } @@ -390,9 +382,13 @@ impl fmt::Display for ShadowErrorV1 { Self::MissingDeliveredCandidate(reference) => { write!(formatter, "missing delivered shadow candidate {reference}") } - Self::PostModelJournal(error) => write!( + Self::PostDurabilityCommit(error) => write!( formatter, - "shadow journal validation failed after the unpublished model transition: {error}" + "shadow model commit failed after WAL durability: {error}" + ), + Self::PostDurabilityJournal(error) => write!( + formatter, + "shadow journal commit failed after WAL durability: {error}" ), Self::Projection(error) => write!(formatter, "{error}"), Self::Poisoned => formatter.write_str("shadow core is poisoned"), @@ -406,8 +402,8 @@ impl Error for ShadowErrorV1 { Self::Wal(error) => Some(error), Self::Codec(error) => Some(error), Self::Carrier(error) => Some(error), - Self::Model(error) => Some(error), - Self::Journal(error) | Self::PostModelJournal(error) => Some(error), + Self::Model(error) | Self::PostDurabilityCommit(error) => Some(error), + Self::Journal(error) | Self::PostDurabilityJournal(error) => Some(error), Self::Projection(error) => Some(error), _ => None, } @@ -516,23 +512,16 @@ pub(crate) struct StarfishRbcDagShadowV1 { journal: WriteAheadJournalV1, wal: ShadowWalV1, candidates: BTreeMap, - application_carriers: BTreeMap>, delivered: BTreeSet, authenticated_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, requested_recoveries: BTreeMap>, projection: CertifiedProjectionModel, - pending_projection_candidates: BTreeSet, projection_rejected: BTreeMap, projected_decisions: BTreeSet, - included_applications: BTreeSet, - highest_projected_consensus_round: RoundNumber, - next_undecided_consensus_round: RoundNumber, - next_local_consensus_round: RoundNumber, pending_projected_vertices: Vec, pending_projection_decisions: Vec, - pending_committed_frontiers: Vec, poisoned: bool, } @@ -569,8 +558,7 @@ impl StarfishRbcDagShadowV1 { let replayed_batches = recovery.batch_count(); let discarded_tail_bytes = recovery.discarded_tail_bytes(); - let mut model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; - model.enable_intrinsic_empty_data_availability(); + let model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; let projection = CertifiedProjectionModel::from_committee_context(committee.clone()); let journal = WriteAheadJournalV1::new(context, own_authority); let mut core = Self { @@ -582,23 +570,16 @@ impl StarfishRbcDagShadowV1 { journal, wal, candidates: BTreeMap::new(), - application_carriers: BTreeMap::new(), delivered: BTreeSet::new(), authenticated_slots: BTreeMap::new(), ordinarily_retained_slots: BTreeMap::new(), slot_candidates: BTreeMap::new(), requested_recoveries: BTreeMap::new(), projection, - pending_projection_candidates: BTreeSet::new(), projection_rejected: BTreeMap::new(), projected_decisions: BTreeSet::new(), - included_applications: BTreeSet::new(), - highest_projected_consensus_round: 0, - next_undecided_consensus_round: 1, - next_local_consensus_round: 1, pending_projected_vertices: Vec::new(), pending_projection_decisions: Vec::new(), - pending_committed_frontiers: Vec::new(), poisoned: false, }; @@ -610,7 +591,6 @@ impl StarfishRbcDagShadowV1 { // not re-emitted as fresh runtime observations after restart. core.pending_projection_decisions.clear(); core.pending_projected_vertices.clear(); - core.pending_committed_frontiers.clear(); let recovery_effects = core .requested_recoveries .iter() @@ -700,10 +680,6 @@ impl StarfishRbcDagShadowV1 { std::mem::take(&mut self.pending_projected_vertices) } - pub(crate) fn drain_committed_frontiers(&mut self) -> Vec { - std::mem::take(&mut self.pending_committed_frontiers) - } - /// Persist the external data-availability predicate for one exact /// application-bearing carrier. Control carriers are available by shape /// and never require this oracle. @@ -723,10 +699,30 @@ impl StarfishRbcDagShadowV1 { } pub(crate) fn application_carriers(&self, application: BlockReference) -> Vec { - self.application_carriers - .get(&application) - .map(|carriers| carriers.iter().copied().collect()) - .unwrap_or_default() + self.candidates + .iter() + .filter_map(|(reference, candidate)| { + candidate + .header() + .application_header() + .is_some_and(|header| header.reference() == application) + .then_some(*reference) + }) + .collect() + } + + /// Application headers with the canonical empty commitment need no + /// transaction reconstruction. Their exact carrier bytes are therefore + /// sufficient data-availability evidence once the carrier is retained. + pub(crate) fn intrinsically_available_applications(&self) -> Vec { + self.candidates + .values() + .filter_map(|candidate| candidate.header().application_header()) + .filter(|header| header.transactions_commitment() == TransactionsCommitment::default()) + .map(RbcCanonicalHeader::reference) + .collect::>() + .into_iter() + .collect() } pub(crate) fn carrier_data_available(&self, reference: BlockReference) -> bool { @@ -922,7 +918,12 @@ impl StarfishRbcDagShadowV1 { } fn next_local_consensus_round(&self) -> RoundNumber { - self.next_local_consensus_round + let snapshot = self.journal.snapshot(); + let mut round: RoundNumber = 1; + while snapshot.consensus_slot(round).is_some() { + round = round.saturating_add(1); + } + round } /// Verify and durably apply an authenticated network envelope for this @@ -1183,25 +1184,20 @@ impl StarfishRbcDagShadowV1 { ) -> Result, ShadowErrorV1> { self.delivered .iter() - .map(|reference| self.delivery_identity(*reference)) + .map(|reference| { + let candidate = self + .candidates + .get(reference) + .ok_or(ShadowErrorV1::MissingDeliveredCandidate(*reference))?; + Ok(ShadowDeliveryIdentityV1::new( + candidate.header().author(), + candidate.header().carrier_round(), + candidate.header().transactions_commitment(), + )) + }) .collect() } - pub(crate) fn delivery_identity( - &self, - reference: BlockReference, - ) -> Result { - let candidate = self - .candidates - .get(&reference) - .ok_or(ShadowErrorV1::MissingDeliveredCandidate(reference))?; - Ok(ShadowDeliveryIdentityV1::new( - candidate.header().author(), - candidate.header().carrier_round(), - candidate.header().transactions_commitment(), - )) - } - /// Exact application headers whose enclosing carriers reached embedded /// RBC delivery. Control-only carrier deliveries are intentionally absent. pub(crate) fn delivered_application_headers( @@ -1210,29 +1206,22 @@ impl StarfishRbcDagShadowV1 { self.delivered .iter() .filter_map(|carrier_reference| { - match self.delivered_application_header(*carrier_reference) { - Ok(Some(application)) => Some(Ok(application)), - Ok(None) => None, - Err(error) => Some(Err(error)), - } + let candidate = match self.candidates.get(carrier_reference) { + Some(candidate) => candidate, + None => { + return Some(Err(ShadowErrorV1::MissingDeliveredCandidate( + *carrier_reference, + ))); + } + }; + candidate + .header() + .application_header() + .map(|header| Ok((*carrier_reference, header.clone()))) }) .collect() } - pub(crate) fn delivered_application_header( - &self, - carrier_reference: BlockReference, - ) -> Result, ShadowErrorV1> { - let candidate = self - .candidates - .get(&carrier_reference) - .ok_or(ShadowErrorV1::MissingDeliveredCandidate(carrier_reference))?; - Ok(candidate - .header() - .application_header() - .map(|header| (carrier_reference, header.clone()))) - } - /// Compare protocol-independent delivery sets. Multiple transaction /// commitments for one `(author, round)` slot make the comparison /// ambiguous instead of being resolved by arrival or reference order. @@ -1272,23 +1261,36 @@ impl StarfishRbcDagShadowV1 { loop { let mut advanced = false; let candidates = self - .pending_projection_candidates + .candidates .iter() - .copied() + .filter_map(|(reference, candidate)| { + candidate + .header() + .consensus_vertex() + .is_some() + .then_some(*reference) + }) .collect::>(); for reference in candidates { + let consensus_round = self + .candidates + .get(&reference) + .and_then(|candidate| candidate.header().consensus_vertex()) + .expect("candidate list contains only consensus vertices") + .consensus_round(); + let vertex_reference = ConsensusVertexReference::new(reference, consensus_round); + if self.projection.is_projected(vertex_reference) + || self.projection_rejected.contains_key(&reference) + { + continue; + } match self.projection.try_project(reference) { Ok(projected) => { - self.pending_projection_candidates.remove(&reference); - self.highest_projected_consensus_round = self - .highest_projected_consensus_round - .max(projected.consensus_round()); self.pending_projected_vertices.push(projected); advanced = true; } Err(error) if projection_error_is_pending(&error) => {} Err(error) => { - self.pending_projection_candidates.remove(&reference); self.projection_rejected.insert(reference, error); } } @@ -1298,125 +1300,32 @@ impl StarfishRbcDagShadowV1 { } } - self.drive_ordered_committer(self.highest_projected_consensus_round); - } - - fn drive_ordered_committer(&mut self, highest_round: RoundNumber) { - let decidable_round = highest_round.saturating_sub(2); - loop { - while self.next_undecided_consensus_round <= decidable_round - && self.has_projection_decision( - self.projection - .leader_slot(self.next_undecided_consensus_round), - ) + let highest_round = self + .candidates + .values() + .filter_map(|candidate| candidate.header().consensus_vertex()) + .map(ConsensusVertexV1::consensus_round) + .max() + .unwrap_or_default(); + for round in 1..=highest_round.saturating_sub(2) { + let slot = self.projection.leader_slot(round); + if self + .projected_decisions + .iter() + .any(|decision| projection_decision_slot(*decision) == slot) { - self.next_undecided_consensus_round = - self.next_undecided_consensus_round.saturating_add(1); - } - if self.next_undecided_consensus_round > decidable_round { - return; + continue; } - let round = self.next_undecided_consensus_round; - let slot = self.projection.leader_slot(round); let Ok(decision) = self.projection.direct_decision(slot) else { - return; + continue; }; - match decision { - ProjectionDecisionV1::DirectCommit { leader } => { - self.commit_projected_anchor(leader); - self.record_projection_decision(decision); - self.next_undecided_consensus_round = round.saturating_add(1); - } - ProjectionDecisionV1::DirectSkip { .. } => { - self.record_projection_decision(decision); - self.next_undecided_consensus_round = round.saturating_add(1); - } - ProjectionDecisionV1::Undecided { .. } => { - let first_anchor_round = round.saturating_add(3); - let later_anchor = - (first_anchor_round..=decidable_round).find_map(|candidate| { - let candidate_slot = self.projection.leader_slot(candidate); - match self.projection.direct_decision(candidate_slot).ok()? { - ProjectionDecisionV1::DirectCommit { leader } => Some(leader), - ProjectionDecisionV1::DirectSkip { .. } - | ProjectionDecisionV1::IndirectCommit { .. } - | ProjectionDecisionV1::IndirectSkip { .. } - | ProjectionDecisionV1::Undecided { .. } => None, - } - }); - let Some(anchor) = later_anchor else { - return; - }; - self.commit_projected_anchor(anchor); - let indirect = self - .projection - .indirect_decision(slot, anchor) - .expect("a clean committed later anchor must decide the older slot"); - self.record_projection_decision(indirect); - self.record_projection_decision(ProjectionDecisionV1::DirectCommit { - leader: anchor, - }); - self.next_undecided_consensus_round = round.saturating_add(1); - } - ProjectionDecisionV1::IndirectCommit { .. } - | ProjectionDecisionV1::IndirectSkip { .. } => { - unreachable!("direct decision returned an indirect result") - } + if matches!(decision, ProjectionDecisionV1::Undecided { .. }) { + continue; } - } - } - - fn has_projection_decision(&self, slot: LeaderSlotV1) -> bool { - self.projected_decisions - .iter() - .any(|decision| projection_decision_slot(*decision) == slot) - } - - fn record_projection_decision(&mut self, decision: ProjectionDecisionV1) { - if self.projected_decisions.insert(decision) { - self.pending_projection_decisions.push(decision); - } - } - - fn commit_projected_anchor(&mut self, anchor: ConsensusVertexReference) { - if self.projection.is_committed_anchor(anchor) { - return; - } - let frontier = self - .projection - .effective_frontier(anchor) - .expect("a committed anchor must be clean and projected") - .to_vec(); - if let Err(error) = self.projection.record_committed_anchor(anchor) { - if self.projection.committed_frontier_dominates(&frontier) { - // The leader is logically committed, but a later anchor used - // for an indirect decision has already output its complete - // carrier prefix. Re-emitting it would regress the frontier. - return; + if self.projected_decisions.insert(decision) { + self.pending_projection_decisions.push(decision); } - panic!("ordered clean anchors must be comparable exact frontiers: {error}"); } - let carriers = self - .model - .apply_frontier(&frontier) - .expect("projection and reducer closed prefixes must agree"); - let applications = carriers - .iter() - .filter_map(|reference| { - self.candidates - .get(reference) - .and_then(|candidate| candidate.header().application_header()) - }) - .filter(|header| self.included_applications.insert(header.reference())) - .cloned() - .collect(); - self.pending_committed_frontiers - .push(CommittedFrontierDeltaV1 { - anchor, - frontier, - carriers, - applications, - }); } fn ensure_live(&self) -> Result<(), ShadowErrorV1> { @@ -1460,28 +1369,20 @@ impl StarfishRbcDagShadowV1 { } fn apply_durable(&mut self, input: ShadowInputV1) -> Result, ShadowErrorV1> { - let (trace, effects) = self.model.apply_input_unpublished(input.model_input())?; - let records = match encode_batch(self.context, self.own_authority, &input, &trace) { - Ok(records) => records, - Err(error) => { - self.poisoned = true; - return Err(error); - } - }; - let journal_events = match journal_transition_events(&self.journal, &input, &trace) { - Ok(events) => events, + let plan = self.model.plan_input(input.model_input())?; + let records = encode_batch(self.context, self.own_authority, &input, plan.trace())?; + let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; + self.wal.append_batch(&records)?; + let effects = match self.model.commit_plan(plan) { + Ok(effects) => effects, Err(error) => { self.poisoned = true; - return Err(error); + return Err(ShadowErrorV1::PostDurabilityCommit(error)); } }; - if let Err(error) = self.journal.apply_batch_unpublished(journal_events) { + if let Err(error) = self.journal.commit_validated_batch(journal_batch) { self.poisoned = true; - return Err(ShadowErrorV1::PostModelJournal(error)); - } - if let Err(error) = self.wal.append_batch(&records) { - self.poisoned = true; - return Err(error.into()); + return Err(ShadowErrorV1::PostDurabilityJournal(error)); } self.record_committed_input(&input, &effects); Ok(effects) @@ -1500,14 +1401,15 @@ impl StarfishRbcDagShadowV1 { self.own_authority, self.committee.committee().len(), )?; - let (trace, effects) = self.model.apply_input_unpublished(input.model_input())?; - if trace != recorded_trace { + let plan = self.model.plan_input(input.model_input())?; + if plan.trace() != recorded_trace { return Err(ShadowErrorV1::TraceMismatch { batch_sequence }); } - let journal_events = journal_transition_events(&self.journal, &input, &trace)?; - if let Err(error) = self.journal.apply_batch_unpublished(journal_events) { + let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; + let effects = self.model.commit_plan(plan)?; + if let Err(error) = self.journal.commit_validated_batch(journal_batch) { self.poisoned = true; - return Err(ShadowErrorV1::PostModelJournal(error)); + return Err(ShadowErrorV1::PostDurabilityJournal(error)); } self.record_committed_input(&input, &effects); Ok(effects) @@ -1587,26 +1489,10 @@ impl StarfishRbcDagShadowV1 { if let Some(candidate) = input.candidate().cloned() { let reference = candidate.reference(); let slot = carrier_slot(reference); - if let Some(vertex) = candidate.header().consensus_vertex() { - self.pending_projection_candidates.insert(reference); - if input.is_local() { - self.next_local_consensus_round = self - .next_local_consensus_round - .max(vertex.consensus_round().saturating_add(1)); - } - } - if let Some(application) = candidate.header().application_header() { - self.application_carriers - .entry(application.reference()) - .or_default() - .insert(reference); - } self.projection .stage_carrier(candidate.clone()) .expect("durably validated carrier must match projection committee"); - if candidate.header().transactions_commitment() == TransactionsCommitment::default() - || input.is_local() - { + if candidate.header().application_header().is_none() || input.is_local() { self.projection .mark_data_available(reference) .expect("staged control carrier is available"); @@ -1851,11 +1737,11 @@ fn decode_authentication( Ok(authentication) } -fn journal_transition_events( +fn validate_journal_transition( journal: &WriteAheadJournalV1, input: &ShadowInputV1, trace: &[ModelTraceEvent], -) -> Result, ShadowErrorV1> { +) -> Result { let mut events = Vec::new(); let context = journal.snapshot().context(); match input { @@ -1981,7 +1867,7 @@ fn journal_transition_events( reference: authenticated.reference(), }); } - Ok(events) + journal.validate_batch(events).map_err(Into::into) } fn encode_batch( @@ -3031,64 +2917,6 @@ mod tests { assert!(!round_is_stale(66, 2)); } - #[test] - fn rejected_far_future_ingress_does_not_poison_the_durable_actor() { - let mut network = TestNetwork::new(); - let author = 1; - let previous = |authority: AuthorityIndex| BlockReference { - authority, - round: 5, - digest: BlockDigest::from([0x90 + authority as u8; 32]), - }; - let candidate = CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 6, - own_prev: previous(author), - weak_parents: [0, 2].into_iter().map(previous).collect(), - transactions_commitment: TransactionsCommitment::default(), - application_header: None, - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: 1, - }, - &network.committee, - ) - .unwrap(); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: author, - keys: &network.keyrings[author as usize], - }, - ) - .unwrap(); - - assert!(matches!( - network.nodes[0].receive_or_retain_from_peer( - &candidate.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - author, - ), - Err(ShadowErrorV1::Model( - ModelError::FutureCarrierOutsideBuffer { - current: 1, - maximum: 5, - actual: 6, - } - )) - )); - assert_eq!(network.nodes[0].wal_counts(), (0, 0)); - network.nodes[0] - .create_local_control_heartbeat(2, true) - .expect("a preflight rejection must leave the actor live"); - assert_eq!(network.nodes[0].wal_counts().0, 1); - } - #[test] fn caller_supplied_relay_provenance_must_be_canonical_for_the_author() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 7aff59da..acd0fbeb 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -40,9 +40,9 @@ use crate::{ storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ - CommittedFrontierDeltaV1, ShadowAuthorizerV1, ShadowDeliveryComparisonV1, - ShadowDeliveryIdentityV1, ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, - ShadowOpenReportV1, ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, + ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, + ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, ShadowOpenReportV1, + ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, }, types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, }; @@ -408,7 +408,6 @@ pub(crate) enum ShadowServiceEventV1 { }, VertexProjected(ConsensusVertexReference), LeaderDecided(ProjectionDecisionV1), - FrontierCommitted(CommittedFrontierDeltaV1), Comparison(ShadowDeliveryComparisonV1), Input { kind: &'static str, @@ -984,7 +983,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( invalidated_by_overload: actor_invalidated_by_overload, pending_local, assigned_applications, - pending_data_availability: BTreeSet::new(), + available_applications: BTreeSet::new(), pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), @@ -1115,7 +1114,7 @@ struct ShadowServiceStateV1 { invalidated_by_overload: Arc>>, pending_local: BTreeMap, assigned_applications: BTreeSet, - pending_data_availability: BTreeSet, + available_applications: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, @@ -1308,15 +1307,6 @@ impl ShadowServiceStateV1 { let carrier_round_advanced = effects .iter() .any(|effect| matches!(effect, ModelEffect::CarrierRoundAdvanced(_))); - let newly_delivered = effects - .iter() - .filter_map(|effect| match effect { - ModelEffect::Delivered(reference) => Some(*reference), - ModelEffect::NeedCarrier { .. } - | ModelEffect::PrefixAdvanced { .. } - | ModelEffect::CarrierRoundAdvanced(_) => None, - }) - .collect::>(); if carrier_round_advanced { self.sync_catch_up = std::mem::take(&mut self.sync_used_in_open_round); } @@ -1343,7 +1333,7 @@ impl ShadowServiceStateV1 { self.pending_recovery.len(), )); self.reconcile_data_availability(); - self.report_new_shadow_deliveries(&newly_delivered); + self.report_new_shadow_deliveries(); self.report_projection_progress(); self.flush_carrier_sync_requests(false); self.emit_clock_state(); @@ -1353,14 +1343,9 @@ impl ShadowServiceStateV1 { if !self.mode.is_autonomous() { return; } - let incoming = std::mem::take(&mut *self.desired_available_applications.lock()); - self.pending_data_availability.extend(incoming); - let pending = self - .pending_data_availability - .iter() - .copied() - .collect::>(); - for application in pending { + let mut desired = self.desired_available_applications.lock().clone(); + desired.extend(self.core.intrinsically_available_applications()); + for application in desired { let carriers = self.core.application_carriers(application); if carriers.is_empty() { continue; @@ -1385,7 +1370,7 @@ impl ShadowServiceStateV1 { .iter() .all(|carrier| self.core.carrier_data_available(*carrier)) { - self.pending_data_availability.remove(&application); + self.available_applications.insert(application); } } } @@ -1397,9 +1382,6 @@ impl ShadowServiceStateV1 { for decision in self.core.drain_projection_decisions() { self.emit(ShadowServiceEventV1::LeaderDecided(decision)); } - for delta in self.core.drain_committed_frontiers() { - self.emit(ShadowServiceEventV1::FrontierCommitted(delta)); - } } fn try_create_autonomous_carrier(&mut self, allow_no_vote: bool) { @@ -1882,42 +1864,41 @@ impl ShadowServiceStateV1 { } } - fn report_new_shadow_deliveries(&mut self, references: &[BlockReference]) { - for reference in references { - let identity = match self.core.delivery_identity(*reference) { - Ok(identity) => identity, - Err(error) => { - self.reject(None, error); - return; - } - }; - if !self.reported_shadow_deliveries.insert(identity) { - continue; + fn report_new_shadow_deliveries(&mut self) { + let identities: BTreeSet<_> = match self.core.delivered_identities() { + Ok(identities) => identities.into_iter().collect(), + Err(error) => { + self.reject(None, error); + return; } - let slot = delivery_slot(&identity); + }; + let new_identities = identities + .difference(&self.reported_shadow_deliveries) + .copied() + .collect::>(); + self.reported_shadow_deliveries = identities; + for identity in &new_identities { + let slot = delivery_slot(identity); if !self.mode.is_autonomous() { self.comparison_backlog.observe_epoch_shadow(slot); } - self.emit(ShadowServiceEventV1::Delivered(identity)); + self.emit(ShadowServiceEventV1::Delivered(*identity)); self.emit_slot_comparison(slot); self.emit_comparison_backlog(); - match self.core.delivered_application_header(*reference) { - Ok(Some((carrier, header))) => { - if self - .reported_application_deliveries - .insert(header.reference()) - { - self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { - carrier, - header, - }); - } - } - Ok(None) => {} - Err(error) => { - self.reject(None, error); - return; - } + } + let applications = match self.core.delivered_application_headers() { + Ok(applications) => applications, + Err(error) => { + self.reject(None, error); + return; + } + }; + for (carrier, header) in applications { + if self + .reported_application_deliveries + .insert(header.reference()) + { + self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header }); } } } @@ -2266,7 +2247,10 @@ fn validate_wire_size( fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { matches!( error, - ShadowErrorV1::Wal(_) | ShadowErrorV1::PostModelJournal(_) | ShadowErrorV1::Poisoned + ShadowErrorV1::Wal(_) + | ShadowErrorV1::PostDurabilityCommit(_) + | ShadowErrorV1::PostDurabilityJournal(_) + | ShadowErrorV1::Poisoned ) } @@ -2480,7 +2464,6 @@ mod tests { open_rounds: &mut [RoundNumber], deliveries: &mut [usize], application_deliveries: &mut [BTreeSet], - committed_frontiers: &mut [Vec], sync_requests: &mut usize, projected_vertices: &mut usize, projected_decisions: &mut usize, @@ -2548,9 +2531,6 @@ mod tests { ShadowServiceEventV1::LeaderDecided(_) => { *projected_decisions = projected_decisions.saturating_add(1); } - ShadowServiceEventV1::FrontierCommitted(delta) => { - committed_frontiers[sender].push(delta); - } ShadowServiceEventV1::Rejected { error, .. } if error.contains("FutureCarrierOutsideBuffer") || error.contains("unexpected shadow response") => {} @@ -2809,7 +2789,6 @@ mod tests { let mut open_rounds = vec![1; n]; let mut deliveries = vec![0; n]; let mut application_deliveries = vec![BTreeSet::new(); n]; - let mut committed_frontiers = vec![Vec::new(); n]; let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; @@ -2830,7 +2809,6 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, - &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, @@ -2845,12 +2823,6 @@ mod tests { ); assert!(open_rounds.iter().all(|round| *round >= 19)); assert!(projected_vertices >= n * 3); - assert!( - committed_frontiers - .iter() - .all(|commits| !commits.is_empty()), - "every node must commit at least one certified frontier: {committed_frontiers:?}" - ); assert!( projected_decisions > 0, "clean projection did not decide: vertices={projected_vertices}, rounds={open_rounds:?}" @@ -2933,7 +2905,6 @@ mod tests { let mut open_rounds = vec![1; N]; let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; - let mut committed_frontiers = vec![Vec::new(); N]; let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; @@ -2943,34 +2914,12 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, - &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, 5, ) .await; - // The first directly committed consensus frontier may predate the - // round-one application deliveries. Advance enough certified carrier - // rounds for a later committed frontier to include that closed prefix. - for fixed_round in 5..=18 { - for handle in &handles { - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - } - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut application_deliveries, - &mut committed_frontiers, - &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, - fixed_round + 1, - ) - .await; - } assert!( application_deliveries @@ -2980,36 +2929,13 @@ mod tests { ); assert!( open_rounds.iter().all(|round| *round >= 5), - "the carrier DAG must keep advancing after application delivery" + "application-critical phase carriers must not wait for a heartbeat tick" ); assert_eq!(sync_requests, 0); assert!( projected_vertices >= N, "empty embedded applications must not stall clean projection" ); - let committed_applications = committed_frontiers - .iter() - .map(|commits| { - commits - .iter() - .flat_map(|delta| delta.applications.iter().map(RbcCanonicalHeader::reference)) - .collect::>() - }) - .collect::>(); - assert!( - committed_applications - .iter() - .all(|applications| applications == &committed_applications[0]), - "equal certified frontiers must output byte-identical application order: {committed_applications:?}" - ); - assert_eq!( - committed_applications[0] - .iter() - .copied() - .collect::>(), - expected, - "the committed frontier closure must output every exact application once; commits={committed_frontiers:?}" - ); drop(events); for handle in &handles { @@ -3162,7 +3088,6 @@ mod tests { let mut open_rounds = vec![1; N]; let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; - let mut committed_frontiers = vec![Vec::new(); N]; let mut sync_requests = 0; let mut projected_vertices = 0; let mut projected_decisions = 0; @@ -3175,7 +3100,6 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, - &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, @@ -3229,7 +3153,6 @@ mod tests { &mut open_rounds, &mut deliveries, &mut application_deliveries, - &mut committed_frontiers, &mut sync_requests, &mut projected_vertices, &mut projected_decisions, diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 7f428984..49728a65 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -24,7 +24,6 @@ use crate::{ runtime::timestamp_utc, sailfish_service::SailfishServiceMessage, starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, - starfish_rbc_dag_shadow::CommittedFrontierDeltaV1, starfish_rbc_dag_shadow_service::StarfishRbcDagShadowServiceHandleV1, starfish_rbc_service::{RbcLocalHeader, RbcServiceHandle}, types::{ @@ -82,7 +81,6 @@ pub struct Syncer { sailfish_tx: Option>, starfish_rbc_service: Option, starfish_rbc_dag_shadow_service: Option, - rbc_dag_frontier_authority: bool, } pub trait SyncerSignals: Send + Sync { @@ -97,13 +95,6 @@ pub trait CommitObserver: Send + Sync { committed_leaders: Vec<(Data, Option)>, ) -> Vec; - fn handle_rbc_dag_commit( - &mut self, - dag_state: &DagState, - anchor: BlockReference, - applications: &[BlockReference], - ) -> Vec; - fn recover_committed( &mut self, committed: AHashSet, @@ -115,7 +106,7 @@ pub trait CommitObserver: Send + Sync { impl Syncer { pub fn new( - mut core: Core, + core: Core, signals: S, commit_observer: C, metrics: Arc, @@ -123,11 +114,7 @@ impl Syncer { sailfish_tx: Option>, starfish_rbc_service: Option, starfish_rbc_dag_shadow_service: Option, - rbc_dag_frontier_authority: bool, ) -> Self { - if rbc_dag_frontier_authority { - core.enable_rbc_dag_application_production(); - } let committee_size = core.committee().len(); let own_stake = core .committee() @@ -149,7 +136,6 @@ impl Syncer { sailfish_tx, starfish_rbc_service, starfish_rbc_dag_shadow_service, - rbc_dag_frontier_authority, } } @@ -276,29 +262,6 @@ impl Syncer { self.core.add_starfish_rbc_reference(reference); self.try_new_block(BlockCreationReason::CertificateEvent); } - - /// Sequence one exact deterministic carrier-frontier delta. In M7 this is - /// the sole application-ordering authority; the legacy Starfish committer - /// remains disabled in this mode. - pub fn apply_starfish_rbc_dag_frontier(&mut self, delta: CommittedFrontierDeltaV1) { - assert!( - self.rbc_dag_frontier_authority, - "RBC-DAG frontier output requires the explicit authority mode" - ); - let applications = delta - .applications - .iter() - .map(RbcCanonicalHeader::reference) - .collect::>(); - let committed = self.commit_observer.handle_rbc_dag_commit( - self.core.dag_state(), - delta.anchor.carrier(), - &applications, - ); - self.core.handle_rbc_dag_committed_delta(committed); - self.try_new_block(BlockCreationReason::PostCommit); - } - /// Store a Sailfish++ timeout certificate in DagState and retry block /// creation (a TC may unblock block creation for the next round). pub fn apply_timeout_cert(&mut self, cert: SailfishTimeoutCert) { @@ -557,9 +520,6 @@ impl Syncer { } pub fn try_new_commit(&mut self) { - if self.rbc_dag_frontier_authority { - return; - } let (newly_committed, any_decided) = self.core.try_commit(); let utc_now = timestamp_utc(); if !newly_committed.is_empty() { @@ -646,15 +606,6 @@ mod tests { Vec::new() } - fn handle_rbc_dag_commit( - &mut self, - _dag_state: &DagState, - _anchor: BlockReference, - _applications: &[BlockReference], - ) -> Vec { - Vec::new() - } - fn recover_committed( &mut self, _committed: AHashSet, @@ -757,7 +708,6 @@ mod tests { None, None, None, - false, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); @@ -857,7 +807,6 @@ mod tests { None, None, None, - false, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index a1a75e52..7f8f02d4 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -377,14 +377,7 @@ mod smoke_tests { let route = prometheus::METRICS_ROUTE; let res = reqwest::get(format! {"http://{address}{route}"}).await?; let string = res.text().await?; - let commit = string.lines().any(|line| { - line.starts_with("commit_index") - && line - .split_whitespace() - .last() - .and_then(|value| value.parse::().ok()) - .is_some_and(|value| value > 0.0) - }); + let commit = string.contains("committed_leaders_total"); Ok(commit) } @@ -606,38 +599,12 @@ mod smoke_tests { let timeout_multiplier = if consensus == "starfish-rbc" { 20 } else { 5 }; let timeout = config::param_defaults::default_leader_timeout() * timeout_multiplier; - if tokio::time::timeout(timeout, await_for_commits(addresses)) - .await - .is_err() - { - let state = validators - .iter() - .map(|validator| { - let metrics = validator.metrics(); - ( - metrics.commit_index.get(), - metrics.starfish_rbc_dag_shadow_carrier_round.get(), - metrics.starfish_rbc_dag_projected_vertices_total.get(), - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get(), - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get(), - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get(), - ) - }) - .collect::>(); - panic!( - "[{consensus}] Failed to gather commits within a few timeouts; \ - per-node (commit index, carrier round, projected, direct commits, frontiers, \ - frontier applications)={state:?}" - ); + tokio::select! { + _ = await_for_commits(addresses) => (), + _ = time::sleep(timeout) => panic!( + "[{consensus}] Failed to gather commits \ + within a few timeouts" + ), } if autonomous_clock { @@ -672,17 +639,7 @@ mod smoke_tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get() - > 0 - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "committed"]) - .get() - > 0 - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["frontier", "application"]) - .get() - > 0) + > 0) && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 }) { break; diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 3cbcc83a..2e6aa3bd 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -19,9 +19,9 @@ only application-header certification authority with but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add `--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Autonomous carriers now create durably locked logical consensus vertices and the clean projection produces -Starfish commit/skip decisions. In embedded-authority mode, committed projected anchors now release -deterministic exact carrier-frontier deltas and the legacy Starfish committer is disabled. The -eventual protocol is new, not a transport option or a version-two alias for `starfish-rbc`. +Starfish commit/skip decisions. Deterministic frontier output still uses the existing Starfish DAG +until milestone seven. The eventual protocol is new, not a transport option or a version-two alias +for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -64,14 +64,12 @@ the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application ca encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. Milestone six adds independently numbered consensus vertices, quorum strong parents, objective Vote/NoVote choices, exact contiguous delivery frontiers, durable local consensus locks, and a live committer -that consumes only RBC-delivered, data-available projected vertices. Milestone seven persists and -reconstructs the corresponding anchor/frontier state, applies exact closed-prefix deltas, and makes -those deltas the sole application output authority. +that consumes only RBC-delivered, data-available projected vertices. -The current authoritative mode changes header certification and application ordering. Direct INIT -is still the application-payload transport and is not a certification vote. Direct ECHO/READY and -the legacy Starfish committer cannot certify or output applications in this mode; only the -carrier-DAG projection's committed frontier deltas can do so. +The current authoritative mode changes header certification and produces certified carrier-DAG +leader decisions. Direct INIT is still the application-payload transport and is not a certification +vote. The legacy Starfish DAG remains only as the temporary application-output scaffold; replacing +its commit/output path with deterministic committed frontier deltas is milestone seven. Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode requires an identical recovered direct-header history, control-only autonomous history reopens @@ -102,10 +100,8 @@ therefore forbidden until the worker has exited; process exit remains safe. A pr same-process restart path needs an operating-system file lock or a fully cancellable storage task. The shadow runtime is not a proof or a production performance implementation. Its default -crash-safe profile intentionally fsyncs every accepted transition. The live fail-stop reducer now -applies preflighted transitions and journal deltas in place, then exposes effects only after WAL -append, avoiding the former full-history clone on every carrier. It still retains unbounded run -history and performs synchronous reducer/storage work. A separate explicit benchmark profile writes the same ordered, +crash-safe profile intentionally fsyncs every accepted transition, and its reference reducer clones +retained model/journal history. A separate explicit benchmark profile writes the same ordered, checksummed frames but syncs them only on clean shutdown; it reports appended and durable records separately and makes no crash-safety claim. This removes the known persistence observer effect without changing the protocol reducer. The runtime also uses a fixed unsolicited-retention window @@ -814,9 +810,8 @@ per peer. Requested historical slots remain recoverable beyond the benchmark-onl retention window. Autonomous benchmark validity is separate from delivery comparison validity. -`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and local-carrier counters -must progress, an idle heartbeat must have been observed, the carrier round and embedded-RBC -delivery count must advance during the measured interval, +`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and heartbeat counters must progress, +the carrier round and embedded-RBC delivery count must advance during the measured interval, recovery must drain, and the reported clock-state/backlog and cross-node skew must remain within the configured empirical guards. These checks establish that the observational carrier plane stayed live and bounded; they are not a partial-synchrony proof. @@ -897,10 +892,7 @@ minimum it must cover: proactive rounds, exact-slot synchronization with idempotent late responses and per-peer rate limiting, multi-round convergence after a validator falls behind, control-only WAL reopen, distinct authentication namespace, and an integration check that direct Starfish-RBC continues - committing while the observational carrier clock advances; and -- composed frontier-authority runs in which every exact application header is embedded-RBC - delivered, all honest nodes release the same deterministic application order without duplicates, - the legacy committer is disabled, and the frontier/application/WAL progress gates remain valid. + committing while the observational carrier clock advances. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -945,7 +937,6 @@ samples. | Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | | Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | -| Frontier output authority (milestone seven) | VALID 10/10 | 950.25 | 2,020.9 ms | 2,082.6 ms | 0.70 MB/s | | Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted @@ -970,15 +961,6 @@ pays for both and its 5.1/8.7-second latency is a red flag rather than a protoco seven must make committed frontier deltas the sole ordering/output path before latency is compared as the complete RBC-DAG protocol. -Milestone seven removes that legacy output gate. Its valid run reached carrier round 795 at every -validator, delivered 79,035 application carriers, released 77,870 exact applications through 1,880 -committed frontiers, projected 19,000 vertices, and ended with zero pending recovery. The in-place -fail-stop reducer, incremental projection indexes, and event-local delivery/data-availability paths -also remove the prototype's history-wide hot-path scans. Block/E2E latency fell by 60.4%/75.9% from -milestone six while throughput recovered to 950.25 tx/s. The remaining 2.02/2.08-second latency is -not the target: follow-up profiling must shorten the certified consensus-round/frontier pipeline -toward the roughly 600 ms unsafe Starfish-MAC reference without weakening RBC or frontier safety. - ## 19. Contained implementation milestones Every milestone is committed separately. @@ -1008,11 +990,8 @@ Every milestone is committed separately. vertices, quorum strong parents, explicit timeout-bound leader choices, contiguous exact delivery frontiers, durable slot/choice locks, and a live clean-only direct committer. Malformed optional vertices do not poison their enclosing carrier. -7. **Frontier linearizer and recovery (implemented within the actor's fail-stop scope):** commit - deterministic frontier deltas, reconstruct prefixes, decisions, and anchors from the ordered - WAL, disable the legacy application committer, and output exact application references once. - Full-validator crash recovery and proof-safe late-node state transfer remain deferred because - the direct payload-transport baseline does not yet persist its own proof-critical RBC state. +7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct + prefixes and anchors, and add late-node and crash/restart tests. 8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, signature Starfish variants, and Sailfish++ before attempting tree dissemination. 9. **Tree dissemination:** distribute vector sub-bundles with redundant routing and a direct timeout From 45ff909dc56026a3159df3c6fbad55846d7ae267 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:40:14 +0200 Subject: [PATCH 55/62] Revert "Add certified RBC-DAG consensus projection" This reverts commit 2275c063623e382b5fac8b3eb9043cc49a4331ed. --- README.md | 28 +- crates/orchestrator/src/main.rs | 8 +- crates/orchestrator/src/measurements.rs | 37 +- crates/starfish-core/src/config.rs | 11 +- crates/starfish-core/src/metrics.rs | 81 +-- crates/starfish-core/src/net_sync.rs | 74 +-- .../src/starfish_rbc_dag/model.rs | 25 +- .../src/starfish_rbc_dag/projection.rs | 31 +- .../src/starfish_rbc_dag_shadow.rs | 520 ++---------------- .../src/starfish_rbc_dag_shadow_service.rs | 159 +----- crates/starfish-core/src/syncer.rs | 9 +- crates/starfish-core/src/validator.rs | 45 +- crates/starfish/src/main.rs | 24 +- docs/starfish-rbc-dag-protocol.md | 43 +- 14 files changed, 128 insertions(+), 967 deletions(-) diff --git a/README.md b/README.md index ea0b261c..30f59c49 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,7 @@ prototype with the limitations documented in its [protocol specification](docs/s **Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate logical projection. Its canonical types, deterministic models, crash journal, comparison shadow, autonomous carrier clock, -authoritative embedded-RBC path, and certified logical consensus projection are implemented. Run -the comparison shadow with +and opt-in authoritative embedded-RBC path are implemented. Run the comparison shadow with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; add `--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact application headers in version-two carriers and make embedded ECHO/READY/delivery their sole @@ -60,11 +59,9 @@ certification authority. Direct INIT remains payload transport, but direct ECHO/ blocks in that mode. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for Starfish-RBC by default); application and encodable phase carriers are emitted immediately. -Autonomous carriers now embed durably locked consensus vertices with quorum strong parents, -explicit Vote/NoVote choices, and exact delivery frontiers. Only RBC-delivered, data-available, -prefix-closed vertices enter the projection or its leader decisions. The existing Starfish DAG is -still the temporary application-output scaffold; the committed frontier linearizer is the next -milestone. Shadow traffic shares the +The current milestone changes certification, not consensus: the existing Starfish DAG still +consumes the embedded deliveries and retains its clean-predecessor proposal gate. The certified +carrier projection and frontier linearizer are the next milestones. Shadow traffic shares the validator's network socket and bandwidth, and deployment requires a homogeneous new-binary committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; `--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean @@ -77,9 +74,8 @@ production retains a short embedded-RBC pipeline tail, so benchmark validation u unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between the cumulative direct and shadow delivery counters. Autonomous runs instead require `starfish_rbc_dag_shadow_clock_valid == 1`, heartbeat/WAL progress, advancing carrier rounds, -in-window embedded-RBC delivery, projected-vertex and clean projected-commit progress, and bounded -clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 -in autonomous mode. +in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports +at most 60 validators in mirror mode and 20 in autonomous mode. A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared @@ -90,22 +86,14 @@ nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Sta | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | | Autonomous comparison, direct RBC authoritative | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | -| Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | The milestone-five run produced 10,722 embedded application deliveries, reached carrier rounds 458–459, and ended with zero pending recovery. It also proves that the earlier 250 ms experimental heartbeat was not the latency cause: application and phase carriers are already event-driven, and using the shared 600 ms timeout did not restore the direct baseline. The remaining slowdown is an expected warning about the transitional architecture—the old direct DAG still serializes proposal -creation on embedded RBC cleanliness. Milestone six now lets the optimistic carrier clock advance -independently and feeds only certified vertices into the logical committer; milestone seven must -remove the remaining legacy output gate by committing deterministic frontier deltas. -The milestone-six run reached carrier rounds 356–359 with 35,506 carrier deliveries, 8,059 -application deliveries, 8,487 projected vertices, 830 clean direct commits, and zero pending -recovery. Its further latency increase is a structural red flag, not a projection-speed claim: -certified decisions currently run alongside the old clean-predecessor/output path, so the benchmark -still pays for both. The next measurement is meaningful only after milestone seven removes that -legacy gate. +creation on embedded RBC cleanliness. Do not tune that obsolete gate; milestone six must let the +optimistic carrier clock advance independently and feed only certified vertices into consensus. The local harness starts its timer after transaction-generator warmup, subtracts warmup counters, and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index e145c583..761eb399 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -63,13 +63,13 @@ pub struct Opts { #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, - /// Enable the persisted Starfish-RBC-DAG research runtime. Without the - /// autonomous flag it remains comparison-only. + /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative + /// shadow. #[clap(long, global = true)] starfish_rbc_dag_shadow: bool, - /// Run the independent Starfish-RBC-DAG carrier clock and certified - /// projection. Requires `--starfish-rbc-dag-shadow`. + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, global = true)] starfish_rbc_dag_autonomous_clock: bool, diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 600aab66..e63c0d0f 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -258,16 +258,13 @@ impl Measurement { x.as_str(), "starfish_rbc_dag_shadow_inputs_total" | "starfish_rbc_dag_shadow_delivery_comparisons_total" - | "starfish_rbc_dag_projection_decisions_total" ) => { match sample.value { prometheus_parse::Value::Counter(value) => { - let shadow_bucket = if matches!( - x.as_str(), - "starfish_rbc_dag_shadow_delivery_comparisons_total" - | "starfish_rbc_dag_projection_decisions_total" - ) { + let shadow_bucket = if x + == "starfish_rbc_dag_shadow_delivery_comparisons_total" + { sample .labels .get("outcome") @@ -294,7 +291,6 @@ impl Measurement { | "starfish_rbc_dag_shadow_wal_durable_batches_total" | "starfish_rbc_dag_shadow_wal_durable_records_total" | "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total" - | "starfish_rbc_dag_projected_vertices_total" ) => { match sample.value { @@ -1163,13 +1159,6 @@ impl MeasurementsCollection { ) && self.scalar_counter_increased( "starfish_rbc_dag_shadow_wal_appended_records_total", *scraper_id, - ) && self.scalar_counter_increased( - "starfish_rbc_dag_projected_vertices_total", - *scraper_id, - ) && self.count_bucket_increased( - "starfish_rbc_dag_projection_decisions_total", - *scraper_id, - "direct_commit", ) && self.scalar_gauge_increased( "starfish_rbc_dag_shadow_carrier_round", *scraper_id, @@ -1648,26 +1637,6 @@ mod test { ..Measurement::default() }, ); - collection.add( - scraper_id, - "starfish_rbc_dag_projected_vertices_total".to_owned(), - Measurement { - timestamp, - count: wal_durable_records, - scalar: wal_durable_records as f64, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_projection_decisions_total".to_owned(), - Measurement { - timestamp, - count_buckets: HashMap::from([("direct_commit".to_owned(), wal_durable_records)]), - count: wal_durable_records, - ..Measurement::default() - }, - ); collection.add( scraper_id, "starfish_rbc_dag_shadow_pending_recovery".to_owned(), diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 45721202..c43c27cf 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -67,13 +67,14 @@ pub struct NodeParameters { /// other protocols. #[serde(default)] pub starfish_rbc_protocol_instance: Option<[u8; 32]>, - /// Run the persisted Starfish-RBC-DAG implementation alongside the - /// legacy direct Starfish-RBC service. Without autonomous mode it is a - /// comparison-only mirror. + /// Run the persisted Starfish-RBC-DAG carrier implementation alongside + /// the authoritative direct Starfish-RBC service. Shadow delivery is + /// observational only and cannot affect the DAG, pacemaker, or commits. #[serde(default)] pub starfish_rbc_dag_shadow: bool, - /// Run an independent carrier clock and certified logical projection. - /// This remains experimental and requires `starfish_rbc_dag_shadow`. + /// Let the non-authoritative Starfish-RBC-DAG shadow create an autonomous + /// optimistic carrier clock. This remains experimental and requires + /// `starfish_rbc_dag_shadow`. #[serde(default)] pub starfish_rbc_dag_autonomous_clock: bool, /// Use embedded carrier ECHO/READY delivery as the certification authority diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index f99b0f4f..5d47fc66 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -41,8 +41,8 @@ pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR: i64 = 4; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; -/// Local-benchmark guards for the autonomous RBC-DAG carrier clock. The -/// round-skew limit matches the executable model's bounded future +/// Local-benchmark guards for the non-authoritative autonomous carrier +/// clock. The round-skew limit matches the executable model's bounded future /// buffer. A healthy clock can transiently retain phase work, but its carrier /// capacity exceeds the two RBC statements generated per admitted value; a /// sixteen-committee backlog therefore leaves generous scheduling headroom @@ -198,8 +198,6 @@ pub struct Metrics { pub starfish_rbc_dag_shadow_admitted_authors: IntGauge, pub starfish_rbc_dag_shadow_admitted_stake: IntGauge, pub starfish_rbc_dag_shadow_buffered_authenticated: IntGauge, - pub starfish_rbc_dag_projected_vertices_total: IntCounter, - pub starfish_rbc_dag_projection_decisions_total: IntCounterVec, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -270,8 +268,6 @@ pub struct AutonomousClockBenchmarkBaseline { accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, - projected_vertices: u64, - projection_decisions: u64, wal_batches: u64, wal_records: u64, carrier_round: i64, @@ -297,8 +293,6 @@ struct AutonomousClockBenchmarkSummary { accepted_heartbeats: u64, delivered_carriers: u64, delivered_applications: u64, - projected_vertices: u64, - projection_decisions: u64, wal_batches: u64, wal_records: u64, pending_recovery: i64, @@ -353,13 +347,6 @@ fn summarize_autonomous_clock_benchmark( .with_label_values(&["delivery", "embedded_application"]) .get() > baseline.delivered_applications) - && metrics.starfish_rbc_dag_projected_vertices_total.get() - > baseline.projected_vertices - && metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get() - > baseline.projection_decisions && metrics .starfish_rbc_dag_shadow_wal_appended_batches_total .get() @@ -416,19 +403,6 @@ fn summarize_autonomous_clock_benchmark( .get() }) .sum(); - let projected_vertices = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_projected_vertices_total.get()) - .sum(); - let projection_decisions = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get() - }) - .sum(); let wal_batches = metrics .iter() .map(|metrics| { @@ -496,8 +470,6 @@ fn summarize_autonomous_clock_benchmark( accepted_heartbeats, delivered_carriers, delivered_applications, - projected_vertices, - projection_decisions, wal_batches, wal_records, pending_recovery, @@ -538,11 +510,6 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "embedded_application"]) .get(), - projected_vertices: self.starfish_rbc_dag_projected_vertices_total.get(), - projection_decisions: self - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get(), wal_batches: self .starfish_rbc_dag_shadow_wal_appended_batches_total .get(), @@ -969,7 +936,7 @@ impl Metrics { .unwrap(), starfish_rbc_dag_shadow_clock_valid: register_int_gauge_with_registry!( "starfish_rbc_dag_shadow_clock_valid", - "State of the autonomous RBC-DAG carrier clock and projection runtime (1 valid, 0 disabled/invalid, -1 starting)", + "State of the non-authoritative autonomous carrier clock (1 valid, 0 disabled/invalid, -1 starting)", registry, ) .unwrap(), @@ -1003,20 +970,6 @@ impl Metrics { registry, ) .unwrap(), - starfish_rbc_dag_projected_vertices_total: register_int_counter_with_registry!( - "starfish_rbc_dag_projected_vertices_total", - "RBC-delivered, data-available consensus vertices admitted to the certified carrier projection", - registry, - ) - .unwrap(), - starfish_rbc_dag_projection_decisions_total: - register_int_counter_vec_with_registry!( - "starfish_rbc_dag_projection_decisions_total", - "Clean-only Starfish leader decisions produced by the certified carrier projection", - &["outcome"], - registry, - ) - .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1759,12 +1712,10 @@ impl Metrics { table.add_row(row![ b->"Clock/WAL progress:", format!( - "heartbeats={}, carrier deliveries={}, application deliveries={}, projected vertices={}, projected commits={}, WAL batches={}, records={}, open rounds={}..{}", + "heartbeats={}, carrier deliveries={}, application deliveries={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, summary.delivered_carriers, summary.delivered_applications, - summary.projected_vertices, - summary.projection_decisions, summary.wal_batches, summary.wal_records, summary.minimum_round, @@ -2281,11 +2232,6 @@ mod tests { metrics .starfish_rbc_dag_shadow_buffered_authenticated .set(2); - metrics.starfish_rbc_dag_projected_vertices_total.inc(); - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .inc(); let gathered = registry.gather(); for name in [ @@ -2308,8 +2254,6 @@ mod tests { "starfish_rbc_dag_shadow_admitted_authors", "starfish_rbc_dag_shadow_admitted_stake", "starfish_rbc_dag_shadow_buffered_authenticated", - "starfish_rbc_dag_projected_vertices_total", - "starfish_rbc_dag_projection_decisions_total", ] { assert!( gathered.iter().any(|family| family.get_name() == name), @@ -2355,11 +2299,6 @@ mod tests { metrics .starfish_rbc_dag_shadow_buffered_authenticated .set(buffered_authenticated); - metrics.starfish_rbc_dag_projected_vertices_total.inc(); - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .inc(); metrics } @@ -2424,11 +2363,6 @@ mod tests { .starfish_rbc_dag_shadow_wal_durable_records_total .inc(); metrics.starfish_rbc_dag_shadow_carrier_round.inc(); - metrics.starfish_rbc_dag_projected_vertices_total.inc(); - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .inc(); } assert!( @@ -2439,7 +2373,7 @@ mod tests { #[test] fn embedded_authority_summary_requires_application_delivery_progress() { - let metrics = [autonomous_clock_metrics(8, 0, 0)]; + let metrics = vec![autonomous_clock_metrics(8, 0, 0)]; let baselines = metrics .iter() .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) @@ -2460,11 +2394,6 @@ mod tests { .starfish_rbc_dag_shadow_wal_appended_records_total .inc(); metrics.starfish_rbc_dag_shadow_carrier_round.inc(); - metrics.starfish_rbc_dag_projected_vertices_total.inc(); - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .inc(); assert!( !summarize_autonomous_clock_benchmark( diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 5ce15418..c400fb5b 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -48,7 +48,7 @@ use crate::{ starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader, RbcCommitteeId, RbcProtocolInstanceId}, starfish_rbc_dag::{ RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, - projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, + storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, @@ -1816,11 +1816,10 @@ impl NetworkSyncer Ok(headers) => Some(headers), Err(error) => { // A partial local history would make delivery comparisons - // meaningless in mirror mode and make autonomous local - // application-origin reconciliation unsafe. Disable the - // RBC-DAG runtime while allowing the legacy path to continue. + // meaningless. Disable the whole observational run while + // allowing authoritative direct RBC to continue. tracing::error!( - "Disabling RBC-DAG runtime because recovered direct headers cannot be reconciled: {error}" + "Disabling non-authoritative RBC-DAG shadow because recovered direct headers cannot be reconciled: {error}" ); None } @@ -1979,7 +1978,9 @@ impl NetworkSyncer Ok((service, events, task)) => (Some(service), Some(events), Some(task)), Err(error) => { invalidate_shadow_run(&metrics); - tracing::error!("Disabling Starfish-RBC-DAG runtime: {error}"); + tracing::error!( + "Disabling non-authoritative Starfish-RBC-DAG shadow: {error}" + ); (None, None, None) } } @@ -2167,17 +2168,6 @@ impl NetworkSyncer .syncer .add_transaction_data(vec![item], DataSource::StarfishRbcPayload) .await; - if let Some(ref shadow) = - event_inner.starfish_rbc_dag_shadow_service - { - if let Err(error) = shadow.application_data_available(block_ref) { - invalidate_shadow_run(&rbc_metrics); - tracing::warn!( - ?block_ref, - "Failed to record RBC-DAG application availability: {error}" - ); - } - } } RbcServiceEvent::Delivered(header) => { if let Some(ref shadow) = @@ -2315,26 +2305,6 @@ impl NetworkSyncer } } } - ShadowServiceEventV1::VertexProjected(reference) => { - shadow_metrics - .starfish_rbc_dag_projected_vertices_total - .inc(); - tracing::debug!(?reference, "RBC-DAG consensus vertex projected"); - } - ShadowServiceEventV1::LeaderDecided(decision) => { - let outcome = match decision { - ProjectionDecisionV1::DirectCommit { .. } => "direct_commit", - ProjectionDecisionV1::DirectSkip { .. } => "direct_skip", - ProjectionDecisionV1::IndirectCommit { .. } => "indirect_commit", - ProjectionDecisionV1::IndirectSkip { .. } => "indirect_skip", - ProjectionDecisionV1::Undecided { .. } => "undecided", - }; - shadow_metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&[outcome]) - .inc(); - tracing::debug!(?decision, "RBC-DAG projected leader decided"); - } ShadowServiceEventV1::ComparisonBacklog { unpaired_direct, unpaired_shadow, @@ -2456,7 +2426,7 @@ impl NetworkSyncer shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } tracing::warn!( - "Rejected RBC-DAG runtime input from {:?}: {}", + "Rejected non-authoritative RBC-DAG shadow input from {:?}: {}", peer, error ); @@ -2467,39 +2437,23 @@ impl NetworkSyncer }); // Start bridge task that forwards reconstructed transaction data to core - let bridge_metrics = metrics.clone(); let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); - let bridge_metrics = bridge_metrics.clone(); handle.spawn(async move { while let Some(items) = decoded_rx.recv().await { // Reconstruction proves we now have the shard data for the // entire batch. - let shard_refs = items - .iter() - .map(|item| item.block_reference) - .collect::>(); + let shard_refs = items.iter().map(|item| item.block_reference).collect(); bridge_inner .cordial_knowledge .send(CordialKnowledgeMessage::DagParts { headers: Vec::new(), - shards: shard_refs.clone(), + shards: shard_refs, }); bridge_inner .syncer .add_transaction_data(items, DataSource::ShardReconstructor) .await; - if let Some(ref shadow) = bridge_inner.starfish_rbc_dag_shadow_service { - for reference in shard_refs { - if let Err(error) = shadow.application_data_available(reference) { - invalidate_shadow_run(&bridge_metrics); - tracing::warn!( - ?reference, - "Failed to record reconstructed RBC-DAG availability: {error}" - ); - } - } - } } }) }); @@ -2854,13 +2808,13 @@ impl NetworkSyncer .await { Ok(Ok(())) => {} - Ok(Err(error)) => { - tracing::warn!("RBC-DAG runtime did not acknowledge shutdown: {error}") - } + Ok(Err(error)) => tracing::warn!( + "Non-authoritative RBC-DAG shadow did not acknowledge shutdown: {error}" + ), Err(_) => { shadow_shutdown_timed_out = true; tracing::warn!( - "Timed out stopping RBC-DAG runtime; detaching it from validator shutdown" + "Timed out stopping non-authoritative RBC-DAG shadow; detaching it from validator shutdown" ); if let Some(task) = rbc_dag_shadow_service_task.as_ref() { task.abort(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index d4f0e64f..88504e35 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -24,7 +24,7 @@ use crate::{ }; use super::{ - AuthenticatedCarrierV1, CandidateCarrierV1, LeaderChoiceV1, LocallyAuthenticatedCarrierV1, + AuthenticatedCarrierV1, CandidateCarrierV1, LocallyAuthenticatedCarrierV1, MAX_PHASE_STATEMENTS_V1, RbcDagCommitteeId, RbcDagContextV1, RbcPhaseStatementV1, carrier_genesis_reference, }; @@ -91,19 +91,6 @@ pub enum ModelTraceEvent { }, /// The local author fixed one exact carrier before authorizing its ECHO. LocalCarrierFixed(BlockReference), - /// The optional consensus vertex became the author's immutable value for - /// its logical consensus round. This follows fixing the enclosing carrier - /// and precedes any outbound exposure. - ConsensusSlotLocked { - consensus_round: RoundNumber, - enclosing_carrier: BlockReference, - }, - /// The local Vote/NoVote choice embedded in the fixed consensus vertex - /// became immutable before the carrier can be exposed. - LeaderChoiceLocked { - consensus_round: RoundNumber, - choice: LeaderChoiceV1, - }, /// Bracha delivery became slot-global and immutable. DeliveryLocked(BlockReference), /// Existing non-durable output retained in its exact reducer order. @@ -675,16 +662,6 @@ impl RbcDagModel { self.preflight_receive(&carrier)?; self.own_fixed.insert(round, reference); log.proof(ModelTraceEvent::LocalCarrierFixed(reference)); - if let Some(vertex) = header.consensus_vertex() { - log.proof(ModelTraceEvent::ConsensusSlotLocked { - consensus_round: vertex.consensus_round(), - enclosing_carrier: reference, - }); - log.proof(ModelTraceEvent::LeaderChoiceLocked { - consensus_round: vertex.consensus_round(), - choice: vertex.leader_choice(), - }); - } for statement in &expected_phase_batch { self.pending_phase_set.remove(statement); } diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 21797741..f0a94206 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -35,7 +35,7 @@ pub struct LeaderSlotV1 { pub round: RoundNumber, } -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ProjectionDecisionV1 { DirectCommit { leader: ConsensusVertexReference, @@ -254,35 +254,6 @@ impl CertifiedProjectionModel { self.vertices.contains_key(&reference) } - /// Deterministic clean values at one logical consensus round. Byzantine - /// equivocations remain visible as distinct exact references. - pub fn projected_values_at_round(&self, round: RoundNumber) -> Vec { - self.vertices_at_round(round) - .map(|(reference, _)| reference) - .collect() - } - - pub fn projected_vertex( - &self, - reference: ConsensusVertexReference, - ) -> Option<&ConsensusVertexV1> { - self.vertices - .get(&reference) - .map(|projected| &projected.vertex) - } - - /// Current exact closed carrier-prefix frontier in authority order. - pub fn closed_frontier(&self) -> DeliveryFrontierV1 { - self.committee - .authorities() - .map(|authority| self.closed_tip(authority)) - .collect() - } - - pub fn projected_vertex_count(&self) -> usize { - self.vertices.len() - } - pub fn slot_values( &self, author: AuthorityIndex, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index e12441e7..a81e0d71 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -1,13 +1,13 @@ // Copyright (c) 2026 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! Durable, single-owner execution for the embedded-RBC Starfish DAG. +//! Durable, single-owner shadow execution for the embedded-RBC Starfish DAG. //! -//! Direct-mirror mode remains observational. Autonomous mode owns carrier -//! admission, embedded RBC, and certified projection decisions; the legacy -//! Starfish DAG is still the temporary application-output scaffold until M7. -//! One [`ShadowWalV1`] batch is one reducer transition. Effects are returned -//! only after that complete batch has reached durable storage. +//! This adapter is deliberately non-authoritative: it consumes the same +//! carrier bytes as the live protocol, persists its own deterministic input +//! and trace log, and reports comparison results without influencing the live +//! protocol. One [`ShadowWalV1`] batch is one reducer transition. Effects are +//! returned only after that complete batch has reached durable storage. use std::{ collections::{BTreeMap, BTreeSet}, @@ -21,17 +21,13 @@ use crate::{ starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ AuthenticatedCarrierV1, CandidateCarrierV1, CarrierAuthenticationV1, CarrierAuthorizerV1, - CarrierHeaderV1Args, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, - LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagError, - RbcPhaseStatementV1, carrier_genesis_reference, + CarrierHeaderV1Args, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, + RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, journal::{ IngressProvenanceV1, JournalErrorV1, JournalEventV1, ValidatedJournalBatchV1, WriteAheadJournalV1, }, model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, - projection::{ - CertifiedProjectionError, CertifiedProjectionModel, LeaderSlotV1, ProjectionDecisionV1, - }, storage::{ MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, ShadowWalSummaryV1, ShadowWalSyncPolicyV1, ShadowWalV1, @@ -51,7 +47,6 @@ const RECORD_AUTHENTICATED_INGRESS: u8 = 0x01; const RECORD_CANDIDATE_RETENTION: u8 = 0x02; const RECORD_CANDIDATE_RECOVERY: u8 = 0x03; const RECORD_LOCAL_OUTBOUND_CONTENT: u8 = 0x04; -const RECORD_DATA_AVAILABLE: u8 = 0x05; const RECORD_MODEL_TRACE: u8 = 0x10; const RECORD_LOCAL_OUTBOUND_SIDECAR: u8 = 0x11; const RECORD_LOCAL_OUTBOUND_EXPOSE: u8 = 0x12; @@ -63,8 +58,6 @@ const TRACE_PHASE_CURSOR_ADVANCED: u8 = 0x03; const TRACE_LOCAL_CARRIER_FIXED: u8 = 0x04; const TRACE_DELIVERY_LOCKED: u8 = 0x05; const TRACE_EFFECT: u8 = 0x06; -const TRACE_CONSENSUS_SLOT_LOCKED: u8 = 0x07; -const TRACE_LEADER_CHOICE_LOCKED: u8 = 0x08; const EFFECT_NEED_CARRIER: u8 = 0x00; const EFFECT_DELIVERED: u8 = 0x01; @@ -266,7 +259,6 @@ pub(crate) enum ShadowCodecErrorV1 { TrailingBytes(usize), InvalidProvenance(u8), InvalidPhase(u8), - InvalidLeaderChoice(u8), InvalidTrace(u8), InvalidEffect(u8), NonCanonicalHolders, @@ -317,7 +309,6 @@ pub(crate) enum ShadowErrorV1 { MissingDeliveredCandidate(BlockReference), PostDurabilityCommit(ModelError), PostDurabilityJournal(JournalErrorV1), - Projection(CertifiedProjectionError), Poisoned, } @@ -390,7 +381,6 @@ impl fmt::Display for ShadowErrorV1 { formatter, "shadow journal commit failed after WAL durability: {error}" ), - Self::Projection(error) => write!(formatter, "{error}"), Self::Poisoned => formatter.write_str("shadow core is poisoned"), } } @@ -404,7 +394,6 @@ impl Error for ShadowErrorV1 { Self::Carrier(error) => Some(error), Self::Model(error) | Self::PostDurabilityCommit(error) => Some(error), Self::Journal(error) | Self::PostDurabilityJournal(error) => Some(error), - Self::Projection(error) => Some(error), _ => None, } } @@ -440,12 +429,6 @@ impl From for ShadowErrorV1 { } } -impl From for ShadowErrorV1 { - fn from(error: CertifiedProjectionError) -> Self { - Self::Projection(error) - } -} - #[derive(Clone)] enum ShadowInputV1 { AuthenticatedIngress { @@ -455,7 +438,6 @@ enum ShadowInputV1 { CandidateRetention(CandidateCarrierV1), CandidateRecovery(CandidateCarrierV1), LocalOutbound(LocallyAuthenticatedCarrierV1), - DataAvailable(BlockReference), } impl ShadowInputV1 { @@ -473,18 +455,14 @@ impl ShadowInputV1 { Self::LocalOutbound(authenticated) => { ModelInputRecord::LocalCarrierFixed(authenticated.clone()) } - Self::DataAvailable(reference) => ModelInputRecord::DataAvailable(*reference), } } - fn candidate(&self) -> Option<&CandidateCarrierV1> { + fn candidate(&self) -> &CandidateCarrierV1 { match self { - Self::AuthenticatedIngress { authenticated, .. } => Some(authenticated.candidate()), - Self::CandidateRetention(candidate) | Self::CandidateRecovery(candidate) => { - Some(candidate) - } - Self::LocalOutbound(authenticated) => Some(authenticated.candidate()), - Self::DataAvailable(_) => None, + Self::AuthenticatedIngress { authenticated, .. } => authenticated.candidate(), + Self::CandidateRetention(candidate) | Self::CandidateRecovery(candidate) => candidate, + Self::LocalOutbound(authenticated) => authenticated.candidate(), } } @@ -498,7 +476,7 @@ struct DecodedRawRecord { payload: Vec, } -/// Synchronous, durable RBC-DAG core. +/// Synchronous, non-authoritative shadow core. /// /// This type has one mutable model, journal, and WAL handle and intentionally /// offers no shared-state wrapper. A caller may move it between threads but @@ -517,11 +495,6 @@ pub(crate) struct StarfishRbcDagShadowV1 { ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, requested_recoveries: BTreeMap>, - projection: CertifiedProjectionModel, - projection_rejected: BTreeMap, - projected_decisions: BTreeSet, - pending_projected_vertices: Vec, - pending_projection_decisions: Vec, poisoned: bool, } @@ -559,7 +532,6 @@ impl StarfishRbcDagShadowV1 { let discarded_tail_bytes = recovery.discarded_tail_bytes(); let model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; - let projection = CertifiedProjectionModel::from_committee_context(committee.clone()); let journal = WriteAheadJournalV1::new(context, own_authority); let mut core = Self { committee, @@ -575,11 +547,6 @@ impl StarfishRbcDagShadowV1 { ordinarily_retained_slots: BTreeMap::new(), slot_candidates: BTreeMap::new(), requested_recoveries: BTreeMap::new(), - projection, - projection_rejected: BTreeMap::new(), - projected_decisions: BTreeSet::new(), - pending_projected_vertices: Vec::new(), - pending_projection_decisions: Vec::new(), poisoned: false, }; @@ -587,10 +554,6 @@ impl StarfishRbcDagShadowV1 { let input = core.decode_batch(batch.records())?; core.apply_replayed(input, batch.records(), batch.sequence())?; } - // Historical decisions are reconstructed to validate replay, but are - // not re-emitted as fresh runtime observations after restart. - core.pending_projection_decisions.clear(); - core.pending_projected_vertices.clear(); let recovery_effects = core .requested_recoveries .iter() @@ -672,65 +635,6 @@ impl StarfishRbcDagShadowV1 { .count() } - pub(crate) fn drain_projection_decisions(&mut self) -> Vec { - std::mem::take(&mut self.pending_projection_decisions) - } - - pub(crate) fn drain_projected_vertices(&mut self) -> Vec { - std::mem::take(&mut self.pending_projected_vertices) - } - - /// Persist the external data-availability predicate for one exact - /// application-bearing carrier. Control carriers are available by shape - /// and never require this oracle. - pub(crate) fn mark_carrier_data_available( - &mut self, - reference: BlockReference, - ) -> Result, ShadowErrorV1> { - self.ensure_live()?; - if self - .model - .lifecycle(&reference) - .is_some_and(|lifecycle| lifecycle.data_available) - { - return Ok(Vec::new()); - } - self.apply_durable(ShadowInputV1::DataAvailable(reference)) - } - - pub(crate) fn application_carriers(&self, application: BlockReference) -> Vec { - self.candidates - .iter() - .filter_map(|(reference, candidate)| { - candidate - .header() - .application_header() - .is_some_and(|header| header.reference() == application) - .then_some(*reference) - }) - .collect() - } - - /// Application headers with the canonical empty commitment need no - /// transaction reconstruction. Their exact carrier bytes are therefore - /// sufficient data-availability evidence once the carrier is retained. - pub(crate) fn intrinsically_available_applications(&self) -> Vec { - self.candidates - .values() - .filter_map(|candidate| candidate.header().application_header()) - .filter(|header| header.transactions_commitment() == TransactionsCommitment::default()) - .map(RbcCanonicalHeader::reference) - .collect::>() - .into_iter() - .collect() - } - - pub(crate) fn carrier_data_available(&self, reference: BlockReference) -> bool { - self.model - .lifecycle(&reference) - .is_some_and(|lifecycle| lifecycle.data_available) - } - pub(crate) fn wal_counts(&self) -> (u64, u64) { (self.wal.batch_count(), self.wal.record_count()) } @@ -756,7 +660,6 @@ impl StarfishRbcDagShadowV1 { round, transactions_commitment, None, - None, creation_time_ns, ) } @@ -766,7 +669,6 @@ impl StarfishRbcDagShadowV1 { round: RoundNumber, transactions_commitment: TransactionsCommitment, application_header: Option, - consensus_vertex: Option, creation_time_ns: TimestampNs, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { self.ensure_live()?; @@ -781,7 +683,7 @@ impl StarfishRbcDagShadowV1 { application_header, data_acknowledgments: Vec::new(), phase_batch: self.model.pending_phase_batch(), - consensus_vertex, + consensus_vertex: None, creation_time_ns, }, &self.committee, @@ -812,18 +714,9 @@ impl StarfishRbcDagShadowV1 { pub(crate) fn create_local_control_heartbeat( &mut self, creation_time_ns: TimestampNs, - allow_no_vote: bool, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { let round = self.model.local_carrier_round(); - let (own_prev, _) = self.model.local_parent_set()?; - let consensus_vertex = self.build_local_consensus_vertex(own_prev, allow_no_vote); - self.create_local_carrier_with_application( - round, - TransactionsCommitment::default(), - None, - consensus_vertex, - creation_time_ns, - ) + self.create_local_carrier(round, TransactionsCommitment::default(), creation_time_ns) } /// Assign one exact direct application header to the currently open @@ -833,99 +726,17 @@ impl StarfishRbcDagShadowV1 { &mut self, application_header: RbcCanonicalHeader, creation_time_ns: TimestampNs, - allow_no_vote: bool, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { let round = self.model.local_carrier_round(); let commitment = application_header.transactions_commitment(); - let (own_prev, _) = self.model.local_parent_set()?; - let consensus_vertex = self.build_local_consensus_vertex(own_prev, allow_no_vote); self.create_local_carrier_with_application( round, commitment, Some(application_header), - consensus_vertex, creation_time_ns, ) } - fn build_local_consensus_vertex( - &self, - own_prev: BlockReference, - allow_no_vote: bool, - ) -> Option { - let consensus_round = self.next_local_consensus_round(); - let (strong_parents, leader_choice) = if consensus_round == 1 { - let strong_parents = self - .committee - .committee() - .authorities() - .map(|authority| { - ConsensusVertexReference::new(carrier_genesis_reference(authority), 0) - }) - .collect::>(); - let leader_author = self.committee.committee().elect_leader(0); - ( - strong_parents, - LeaderChoiceV1::Vote { - leader: ConsensusVertexReference::new( - carrier_genesis_reference(leader_author), - 0, - ), - }, - ) - } else { - let parent_round = consensus_round - 1; - let mut by_author = BTreeMap::new(); - for parent in self.projection.projected_values_at_round(parent_round) { - by_author.entry(parent.author()).or_insert(parent); - } - let own_parent = *by_author.get(&self.own_authority)?; - let stake = by_author - .keys() - .filter_map(|authority| self.committee.committee().get_stake(*authority)) - .fold(0, Stake::saturating_add); - if stake < self.committee.committee().quorum_threshold() { - return None; - } - let leader_author = self.committee.committee().elect_leader(parent_round); - let leader_choice = match by_author.get(&leader_author).copied() { - Some(leader) => LeaderChoiceV1::Vote { leader }, - None if allow_no_vote => LeaderChoiceV1::NoVote { - leader_author, - leader_round: parent_round, - }, - None => return None, - }; - let strong_parents = by_author.into_values().collect::>(); - debug_assert!(strong_parents.contains(&own_parent)); - (strong_parents, leader_choice) - }; - - let mut delivery_frontier = self.projection.closed_frontier(); - let own_entry = (own_prev.round != 0).then_some(own_prev); - // The enclosing carrier is not clean yet, so its immediate physical - // predecessor may be ahead of today's closed tip. The immutable - // declaration names that exact predecessor; projection later proves - // the intervening self-chain closed before accepting this vertex. - delivery_frontier.get(self.own_authority as usize)?; - delivery_frontier[self.own_authority as usize] = own_entry; - Some(ConsensusVertexV1::new( - consensus_round, - strong_parents, - delivery_frontier, - leader_choice, - )) - } - - fn next_local_consensus_round(&self) -> RoundNumber { - let snapshot = self.journal.snapshot(); - let mut round: RoundNumber = 1; - while snapshot.consensus_slot(round).is_some() { - round = round.saturating_add(1); - } - round - } - /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -1169,7 +980,8 @@ impl StarfishRbcDagShadowV1 { round: candidate.header().carrier_round(), transactions_commitment: candidate.header().transactions_commitment(), creation_time_ns: candidate.header().creation_time_ns(), - control_shape: candidate.header().data_acknowledgments().is_empty(), + control_shape: candidate.header().data_acknowledgments().is_empty() + && candidate.header().consensus_vertex().is_none(), application: candidate .header() .application_header() @@ -1257,77 +1069,6 @@ impl StarfishRbcDagShadowV1 { Ok(self.wal.shutdown()?) } - fn drive_certified_projection(&mut self) { - loop { - let mut advanced = false; - let candidates = self - .candidates - .iter() - .filter_map(|(reference, candidate)| { - candidate - .header() - .consensus_vertex() - .is_some() - .then_some(*reference) - }) - .collect::>(); - for reference in candidates { - let consensus_round = self - .candidates - .get(&reference) - .and_then(|candidate| candidate.header().consensus_vertex()) - .expect("candidate list contains only consensus vertices") - .consensus_round(); - let vertex_reference = ConsensusVertexReference::new(reference, consensus_round); - if self.projection.is_projected(vertex_reference) - || self.projection_rejected.contains_key(&reference) - { - continue; - } - match self.projection.try_project(reference) { - Ok(projected) => { - self.pending_projected_vertices.push(projected); - advanced = true; - } - Err(error) if projection_error_is_pending(&error) => {} - Err(error) => { - self.projection_rejected.insert(reference, error); - } - } - } - if !advanced { - break; - } - } - - let highest_round = self - .candidates - .values() - .filter_map(|candidate| candidate.header().consensus_vertex()) - .map(ConsensusVertexV1::consensus_round) - .max() - .unwrap_or_default(); - for round in 1..=highest_round.saturating_sub(2) { - let slot = self.projection.leader_slot(round); - if self - .projected_decisions - .iter() - .any(|decision| projection_decision_slot(*decision) == slot) - { - continue; - } - let Ok(decision) = self.projection.direct_decision(slot) else { - continue; - }; - if matches!(decision, ProjectionDecisionV1::Undecided { .. }) { - continue; - } - if self.projected_decisions.insert(decision) { - self.pending_projection_decisions.push(decision); - } - } - } - fn ensure_live(&self) -> Result<(), ShadowErrorV1> { if self.poisoned || self.wal.is_poisoned() { Err(ShadowErrorV1::Poisoned) @@ -1420,12 +1161,10 @@ impl StarfishRbcDagShadowV1 { input: &ShadowInputV1, batch_sequence: u64, ) -> Result<(), ShadowErrorV1> { - let reference = input.candidate().map(CandidateCarrierV1::reference); - let slot = reference.map(carrier_slot); + let reference = input.candidate().reference(); + let slot = carrier_slot(reference); let violation = match input { ShadowInputV1::AuthenticatedIngress { .. } => { - let reference = reference.expect("authenticated ingress has a candidate"); - let slot = slot.expect("authenticated ingress has a slot"); if round_is_stale(self.model.local_carrier_round(), reference.round) { Some("stale authenticated ingress") } else if self.authenticated_slots.contains_key(&slot) { @@ -1435,8 +1174,6 @@ impl StarfishRbcDagShadowV1 { } } ShadowInputV1::CandidateRetention(_) => { - let reference = reference.expect("candidate retention has a candidate"); - let slot = slot.expect("candidate retention has a slot"); if round_is_stale(self.model.local_carrier_round(), reference.round) { Some("stale candidate retention") } else if self.candidates.contains_key(&reference) @@ -1449,8 +1186,6 @@ impl StarfishRbcDagShadowV1 { } } ShadowInputV1::CandidateRecovery(_) => { - let reference = reference.expect("candidate recovery has a candidate"); - let slot = slot.expect("candidate recovery has a slot"); let limit = self .committee .committee() @@ -1472,9 +1207,6 @@ impl StarfishRbcDagShadowV1 { } } ShadowInputV1::LocalOutbound(_) => None, - ShadowInputV1::DataAvailable(reference) => { - (!self.candidates.contains_key(reference)).then_some("unknown available carrier") - } }; if let Some(reason) = violation { return Err(ShadowErrorV1::ReplayPolicyViolation { @@ -1486,41 +1218,26 @@ impl StarfishRbcDagShadowV1 { } fn record_committed_input(&mut self, input: &ShadowInputV1, effects: &[ModelEffect]) { - if let Some(candidate) = input.candidate().cloned() { - let reference = candidate.reference(); - let slot = carrier_slot(reference); - self.projection - .stage_carrier(candidate.clone()) - .expect("durably validated carrier must match projection committee"); - if candidate.header().application_header().is_none() || input.is_local() { - self.projection - .mark_data_available(reference) - .expect("staged control carrier is available"); + let candidate = input.candidate().clone(); + let reference = candidate.reference(); + let slot = carrier_slot(reference); + self.candidates.insert(reference, candidate); + self.slot_candidates + .entry(slot) + .or_default() + .insert(reference); + match input { + ShadowInputV1::AuthenticatedIngress { .. } | ShadowInputV1::LocalOutbound(_) => { + self.authenticated_slots.entry(slot).or_insert(reference); } - self.candidates.insert(reference, candidate); - self.slot_candidates - .entry(slot) - .or_default() - .insert(reference); - match input { - ShadowInputV1::AuthenticatedIngress { .. } | ShadowInputV1::LocalOutbound(_) => { - self.authenticated_slots.entry(slot).or_insert(reference); - } - ShadowInputV1::CandidateRetention(_) => { - self.ordinarily_retained_slots - .entry(slot) - .or_insert(reference); - } - ShadowInputV1::CandidateRecovery(_) => {} - ShadowInputV1::DataAvailable(_) => unreachable!("handled without a candidate"), + ShadowInputV1::CandidateRetention(_) => { + self.ordinarily_retained_slots + .entry(slot) + .or_insert(reference); } - self.requested_recoveries.remove(&reference); - } - if let ShadowInputV1::DataAvailable(reference) = input { - self.projection - .mark_data_available(*reference) - .expect("model accepted availability only for a staged carrier"); + ShadowInputV1::CandidateRecovery(_) => {} } + self.requested_recoveries.remove(&reference); for effect in effects { match effect { ModelEffect::NeedCarrier { target, holders } => { @@ -1529,14 +1246,10 @@ impl StarfishRbcDagShadowV1 { ModelEffect::Delivered(delivered) => { self.delivered.insert(*delivered); self.requested_recoveries.remove(delivered); - self.projection - .mark_delivered(*delivered) - .expect("model delivery must name a staged carrier"); } ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} } } - self.drive_certified_projection(); } fn decode_batch(&self, records: &[Vec]) -> Result { @@ -1582,13 +1295,6 @@ impl StarfishRbcDagShadowV1 { Ok(ShadowInputV1::CandidateRecovery(candidate)) } } - RECORD_DATA_AVAILABLE => { - ensure_trace_tail(&decoded[1..])?; - let mut payload = RawDecoder::new(&decoded[0].payload); - let reference = payload.read_reference()?; - payload.finish()?; - Ok(ShadowInputV1::DataAvailable(reference)) - } RECORD_LOCAL_OUTBOUND_CONTENT => { if decoded.len() != 4 { return Err(ShadowErrorV1::InvalidBatch( @@ -1767,15 +1473,9 @@ fn validate_journal_transition( candidate: authenticated.candidate().clone(), }); } - ShadowInputV1::DataAvailable(_) => {} } - let local_reference = input.is_local().then(|| { - input - .candidate() - .expect("local input has candidate") - .reference() - }); + let local_reference = input.is_local().then(|| input.candidate().reference()); for entry in trace { let event = match entry { ModelTraceEvent::AdmissionLocked(target) if Some(*target) == local_reference => { @@ -1830,22 +1530,6 @@ fn validate_journal_transition( context, reference: *reference, }), - ModelTraceEvent::ConsensusSlotLocked { - consensus_round, - enclosing_carrier, - } => Some(JournalEventV1::LockConsensusSlot { - context, - consensus_round: *consensus_round, - enclosing_carrier: *enclosing_carrier, - }), - ModelTraceEvent::LeaderChoiceLocked { - consensus_round, - choice, - } => Some(JournalEventV1::LockLeaderChoice { - context, - consensus_round: *consensus_round, - choice: *choice, - }), ModelTraceEvent::DeliveryLocked(target) => Some(JournalEventV1::LockDelivery { context, target: *target, @@ -1923,16 +1607,6 @@ fn encode_batch( &authenticated.candidate().canonical_wire_bytes()?, )?); } - ShadowInputV1::DataAvailable(reference) => { - let mut payload = Vec::new(); - push_reference(&mut payload, *reference); - records.push(encode_raw_record( - context, - own_authority, - RECORD_DATA_AVAILABLE, - &payload, - )?); - } } records.push(encode_raw_record( context, @@ -2149,22 +1823,6 @@ fn encode_trace(trace: &ModelTraceEvent) -> Result, ShadowCodecErrorV1> bytes.push(TRACE_LOCAL_CARRIER_FIXED); push_reference(&mut bytes, *reference); } - ModelTraceEvent::ConsensusSlotLocked { - consensus_round, - enclosing_carrier, - } => { - bytes.push(TRACE_CONSENSUS_SLOT_LOCKED); - bytes.extend_from_slice(&consensus_round.to_be_bytes()); - push_reference(&mut bytes, *enclosing_carrier); - } - ModelTraceEvent::LeaderChoiceLocked { - consensus_round, - choice, - } => { - bytes.push(TRACE_LEADER_CHOICE_LOCKED); - bytes.extend_from_slice(&consensus_round.to_be_bytes()); - push_leader_choice(&mut bytes, *choice); - } ModelTraceEvent::DeliveryLocked(reference) => { bytes.push(TRACE_DELIVERY_LOCKED); push_reference(&mut bytes, *reference); @@ -2197,14 +1855,6 @@ fn decode_trace( next_index: decoder.read_u32()? as usize, }, TRACE_LOCAL_CARRIER_FIXED => ModelTraceEvent::LocalCarrierFixed(decoder.read_reference()?), - TRACE_CONSENSUS_SLOT_LOCKED => ModelTraceEvent::ConsensusSlotLocked { - consensus_round: decoder.read_u32()?, - enclosing_carrier: decoder.read_reference()?, - }, - TRACE_LEADER_CHOICE_LOCKED => ModelTraceEvent::LeaderChoiceLocked { - consensus_round: decoder.read_u32()?, - choice: decoder.read_leader_choice()?, - }, TRACE_DELIVERY_LOCKED => ModelTraceEvent::DeliveryLocked(decoder.read_reference()?), TRACE_EFFECT => ModelTraceEvent::Effect(decoder.read_effect(committee_size)?), other => return Err(ShadowCodecErrorV1::InvalidTrace(other)), @@ -2255,28 +1905,6 @@ fn push_phase(bytes: &mut Vec, statement: RbcPhaseStatementV1) { } } -fn push_consensus_reference(bytes: &mut Vec, reference: ConsensusVertexReference) { - push_reference(bytes, reference.carrier()); - bytes.extend_from_slice(&reference.consensus_round().to_be_bytes()); -} - -fn push_leader_choice(bytes: &mut Vec, choice: LeaderChoiceV1) { - match choice { - LeaderChoiceV1::Vote { leader } => { - bytes.push(0); - push_consensus_reference(bytes, leader); - } - LeaderChoiceV1::NoVote { - leader_author, - leader_round, - } => { - bytes.push(1); - bytes.extend_from_slice(&leader_author.to_be_bytes()); - bytes.extend_from_slice(&leader_round.to_be_bytes()); - } - } -} - fn encode_provenance(bytes: &mut Vec, provenance: IngressProvenanceV1) { match provenance { IngressProvenanceV1::DirectFromAuthor => bytes.push(PROVENANCE_DIRECT), @@ -2357,30 +1985,6 @@ fn ambiguous_slots( .collect() } -fn projection_error_is_pending(error: &CertifiedProjectionError) -> bool { - matches!( - error, - CertifiedProjectionError::CarrierNotDelivered(_) - | CertifiedProjectionError::CarrierDataUnavailable(_) - | CertifiedProjectionError::CarrierOutsideClosedPrefix(_) - | CertifiedProjectionError::MissingStrongParent(_) - | CertifiedProjectionError::FrontierNotClosed { .. } - ) -} - -fn projection_decision_slot(decision: ProjectionDecisionV1) -> LeaderSlotV1 { - match decision { - ProjectionDecisionV1::DirectCommit { leader } - | ProjectionDecisionV1::IndirectCommit { leader, .. } => LeaderSlotV1 { - author: leader.author(), - round: leader.consensus_round(), - }, - ProjectionDecisionV1::DirectSkip { slot } - | ProjectionDecisionV1::IndirectSkip { slot, .. } - | ProjectionDecisionV1::Undecided { slot } => slot, - } -} - struct RawDecoder<'a> { bytes: &'a [u8], position: usize, @@ -2446,25 +2050,6 @@ impl<'a> RawDecoder<'a> { } } - fn read_consensus_reference(&mut self) -> Result { - let carrier = self.read_reference()?; - let consensus_round = self.read_u32()?; - Ok(ConsensusVertexReference::new(carrier, consensus_round)) - } - - fn read_leader_choice(&mut self) -> Result { - match self.read_u8()? { - 0 => Ok(LeaderChoiceV1::Vote { - leader: self.read_consensus_reference()?, - }), - 1 => Ok(LeaderChoiceV1::NoVote { - leader_author: self.read_u16()?, - leader_round: self.read_u32()?, - }), - other => Err(ShadowCodecErrorV1::InvalidLeaderChoice(other)), - } - } - fn read_effect(&mut self, committee_size: usize) -> Result { match self.read_u8()? { EFFECT_NEED_CARRIER => { @@ -2670,7 +2255,7 @@ mod tests { assert_eq!(node.local_outbound_envelope(1), None); let before = node.wal_counts(); - let (heartbeat, effects) = node.create_local_control_heartbeat(123, true).unwrap(); + let (heartbeat, effects) = node.create_local_control_heartbeat(123).unwrap(); assert!(effects.is_empty()); assert_eq!(node.wal_counts().0, before.0 + 1); let candidate = decode_candidate( @@ -2688,19 +2273,7 @@ mod tests { assert_eq!(candidate.header().creation_time_ns(), 123); assert!(candidate.header().data_acknowledgments().is_empty()); assert!(candidate.header().phase_batch().is_empty()); - let vertex = candidate - .header() - .consensus_vertex() - .expect("first autonomous heartbeat carries the genesis projection"); - assert_eq!(vertex.consensus_round(), 1); - assert_eq!( - node.journal.snapshot().consensus_slot(1), - Some(heartbeat.reference()) - ); - assert_eq!( - node.journal.snapshot().leader_choice(1), - Some(vertex.leader_choice()) - ); + assert!(candidate.header().consensus_vertex().is_none()); assert_eq!(node.local_outbound_envelope(1), Some(heartbeat.clone())); assert_eq!(node.local_outbound_envelope(2), None); assert_eq!( @@ -2716,7 +2289,7 @@ mod tests { let durable_counts = node.wal_counts(); assert!(matches!( - node.create_local_control_heartbeat(124, true), + node.create_local_control_heartbeat(124), Err(ShadowErrorV1::Model(ModelError::LocalCarrierAlreadyFixed( 1 ))) @@ -2732,7 +2305,7 @@ mod tests { fn autonomous_control_heartbeat_advances_sequentially_and_reopens_exact_bytes() { let mut network = TestNetwork::new(); let first = network.nodes[0] - .create_local_control_heartbeat(1_000, true) + .create_local_control_heartbeat(1_000) .unwrap() .0; for author in [1, 2] { @@ -2761,7 +2334,7 @@ mod tests { assert_eq!(network.nodes[0].current_round_admitted_author_count(), 0); let second = network.nodes[0] - .create_local_control_heartbeat(2_000, true) + .create_local_control_heartbeat(2_000) .unwrap() .0; let second_candidate = decode_candidate( @@ -2801,15 +2374,6 @@ mod tests { assert!(!restarted.can_create_carrier()); assert_eq!(restarted.local_outbound_envelope(1), Some(first)); assert_eq!(restarted.local_outbound_envelope(2), Some(second)); - let first_reference = restarted - .local_outbound_envelope(1) - .expect("round one survived restart") - .reference(); - assert_eq!( - restarted.journal.snapshot().consensus_slot(1), - Some(first_reference) - ); - assert!(restarted.journal.snapshot().leader_choice(1).is_some()); } #[test] diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index acd0fbeb..0f3b784d 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! Async network adapter for the persisted RBC-DAG research runtime. +//! Async, non-authoritative network adapter for the persisted RBC-DAG shadow. use std::{ collections::{BTreeMap, BTreeSet}, @@ -33,10 +33,8 @@ use crate::{ }, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ - ConsensusVertexReference, MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, - RbcDagContextV1, + MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, model::{ModelEffect, ModelError}, - projection::ProjectionDecisionV1, storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ @@ -70,10 +68,9 @@ const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_milli /// Runtime role of the persisted carrier actor. /// /// Mirror mode preserves milestone three's one-to-one comparison against -/// direct Starfish-RBC headers. Autonomous mode opens an independent carrier -/// clock and owns embedded-RBC certification plus clean projection decisions. -/// It deliberately does not call the legacy core dispatcher: replacing the -/// temporary application-output scaffold is the M7 boundary. +/// direct Starfish-RBC headers. Autonomous mode opens an independent, +/// heartbeat-only carrier clock. It remains observational: neither mode can +/// call the core dispatcher or mutate authoritative consensus state. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ShadowServiceModeV1 { DirectMirror, @@ -147,7 +144,6 @@ enum ShadowServiceMessageV1 { TopologyChanged, RetryRecovery, HeartbeatTick, - DataAvailabilityChanged, Shutdown(oneshot::Sender>), } @@ -161,7 +157,6 @@ pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { mode: ShadowServiceModeV1, desired_topology: Arc>>, desired_direct_deliveries: Arc>>, - desired_available_applications: Arc>>, invalidated_by_overload: Arc>>, } @@ -285,34 +280,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { } } - pub(crate) fn application_data_available( - &self, - application: BlockReference, - ) -> Result<(), ShadowServiceErrorV1> { - if !self.mode.is_autonomous() { - return Ok(()); - } - if application.authority as usize >= self.committee_size { - return Err(ShadowServiceErrorV1::UnknownAuthority( - application.authority, - )); - } - if !self - .desired_available_applications - .lock() - .insert(application) - { - return Ok(()); - } - match self - .sender - .try_send(ShadowServiceMessageV1::DataAvailabilityChanged) - { - Ok(()) | Err(TrySendError::Full(_)) => Ok(()), - Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), - } - } - pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { self.update_peer(peer, true) } @@ -374,7 +341,6 @@ impl ShadowServiceMessageV1 { Self::TopologyChanged => "topology_changed", Self::RetryRecovery => "recovery_retry", Self::HeartbeatTick => "heartbeat_tick", - Self::DataAvailabilityChanged => "data_availability_changed", Self::Shutdown(_) => "shutdown", } } @@ -406,8 +372,6 @@ pub(crate) enum ShadowServiceEventV1 { carrier: BlockReference, header: RbcCanonicalHeader, }, - VertexProjected(ConsensusVertexReference), - LeaderDecided(ProjectionDecisionV1), Comparison(ShadowDeliveryComparisonV1), Input { kind: &'static str, @@ -697,7 +661,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let (event_tx, event_rx) = mpsc::channel(SHADOW_SERVICE_EVENT_CAPACITY_V1); let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); - let desired_available_applications = Arc::new(Mutex::new(BTreeSet::new())); let invalidated_by_overload = Arc::new(Mutex::new(None)); let retry_notification_pending = Arc::new(AtomicBool::new(false)); let heartbeat_notification_pending = Arc::new(AtomicBool::new(false)); @@ -759,7 +722,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let startup_events = event_tx.clone(); let actor_desired_topology = Arc::clone(&desired_topology); let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); - let actor_desired_available_applications = Arc::clone(&desired_available_applications); let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); let actor_retry_notification_pending = Arc::clone(&retry_notification_pending); let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); @@ -978,12 +940,10 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( connected: BTreeSet::new(), desired_topology: actor_desired_topology, desired_direct_deliveries: actor_desired_direct_deliveries, - desired_available_applications: actor_desired_available_applications, observed_topology: BTreeMap::new(), invalidated_by_overload: actor_invalidated_by_overload, pending_local, assigned_applications, - available_applications: BTreeSet::new(), pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), @@ -1027,7 +987,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( mode, desired_topology, desired_direct_deliveries, - desired_available_applications, invalidated_by_overload, }, event_rx, @@ -1109,12 +1068,10 @@ struct ShadowServiceStateV1 { connected: BTreeSet, desired_topology: Arc>>, desired_direct_deliveries: Arc>>, - desired_available_applications: Arc>>, observed_topology: BTreeMap, invalidated_by_overload: Arc>>, pending_local: BTreeMap, assigned_applications: BTreeSet, - available_applications: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, @@ -1332,59 +1289,12 @@ impl ShadowServiceStateV1 { self.emit(ShadowServiceEventV1::PendingRecovery( self.pending_recovery.len(), )); - self.reconcile_data_availability(); self.report_new_shadow_deliveries(); - self.report_projection_progress(); self.flush_carrier_sync_requests(false); self.emit_clock_state(); } - fn reconcile_data_availability(&mut self) { - if !self.mode.is_autonomous() { - return; - } - let mut desired = self.desired_available_applications.lock().clone(); - desired.extend(self.core.intrinsically_available_applications()); - for application in desired { - let carriers = self.core.application_carriers(application); - if carriers.is_empty() { - continue; - } - for carrier in &carriers { - if self.core.carrier_data_available(*carrier) { - continue; - } - let before = self.core.wal_counts(); - match self.core.mark_carrier_data_available(*carrier) { - Ok(effects) => { - self.report_wal_delta(before); - self.process_effects(effects); - } - Err(error) => { - self.mark_fatal(error); - return; - } - } - } - if carriers - .iter() - .all(|carrier| self.core.carrier_data_available(*carrier)) - { - self.available_applications.insert(application); - } - } - } - - fn report_projection_progress(&mut self) { - for projected in self.core.drain_projected_vertices() { - self.emit(ShadowServiceEventV1::VertexProjected(projected)); - } - for decision in self.core.drain_projection_decisions() { - self.emit(ShadowServiceEventV1::LeaderDecided(decision)); - } - } - - fn try_create_autonomous_carrier(&mut self, allow_no_vote: bool) { + fn try_create_autonomous_carrier(&mut self) { if !self.mode.is_autonomous() || !self.core.can_create_carrier() { self.emit_clock_state(); return; @@ -1402,11 +1312,8 @@ impl ShadowServiceStateV1 { Some(application) => self.core.create_local_application_carrier( application.application_header.clone(), creation_time_ns, - allow_no_vote, ), - None => self - .core - .create_local_control_heartbeat(creation_time_ns, allow_no_vote), + None => self.core.create_local_control_heartbeat(creation_time_ns), }; match result { Ok((envelope, effects)) => { @@ -1456,7 +1363,7 @@ impl ShadowServiceStateV1 { fn drive_autonomous_catch_up(&mut self) { while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(true); + self.try_create_autonomous_carrier(); if self.core.local_carrier_round() == round_before { break; } @@ -1559,7 +1466,7 @@ impl ShadowServiceStateV1 { && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(false); + self.try_create_autonomous_carrier(); if self.core.local_carrier_round() == round_before { break; } @@ -2171,10 +2078,7 @@ fn run_shadow_service( state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); } - ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(true), - ShadowServiceMessageV1::DataAvailabilityChanged => { - state.reconcile_data_availability(); - } + ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(), ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); @@ -2465,8 +2369,6 @@ mod tests { deliveries: &mut [usize], application_deliveries: &mut [BTreeSet], sync_requests: &mut usize, - projected_vertices: &mut usize, - projected_decisions: &mut usize, target_open_round: RoundNumber, ) { timeout(EVENT_TIMEOUT, async { @@ -2525,12 +2427,6 @@ mod tests { } => { application_deliveries[sender].insert(header.reference()); } - ShadowServiceEventV1::VertexProjected(_) => { - *projected_vertices = projected_vertices.saturating_add(1); - } - ShadowServiceEventV1::LeaderDecided(_) => { - *projected_decisions = projected_decisions.saturating_add(1); - } ShadowServiceEventV1::Rejected { error, .. } if error.contains("FutureCarrierOutsideBuffer") || error.contains("unexpected shadow response") => {} @@ -2790,8 +2686,6 @@ mod tests { let mut deliveries = vec![0; n]; let mut application_deliveries = vec![BTreeSet::new(); n]; let mut sync_requests = 0; - let mut projected_vertices = 0; - let mut projected_decisions = 0; for (authority, handle) in handles.iter().enumerate() { for peer in 0..n { if peer != authority { @@ -2799,7 +2693,7 @@ mod tests { } } } - for fixed_round in 1..=18 { + for fixed_round in 1..=6 { for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -2810,8 +2704,6 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, fixed_round + 1, ) .await; @@ -2821,12 +2713,7 @@ mod tests { deliveries.iter().all(|count| *count > 0), "every node must RBC-deliver mature heartbeat carriers: {deliveries:?}" ); - assert!(open_rounds.iter().all(|round| *round >= 19)); - assert!(projected_vertices >= n * 3); - assert!( - projected_decisions > 0, - "clean projection did not decide: vertices={projected_vertices}, rounds={open_rounds:?}" - ); + assert!(open_rounds.iter().all(|round| *round >= 7)); assert_eq!( sync_requests, 0, "healthy proactive rounds must not trigger repair polling" @@ -2888,11 +2775,8 @@ mod tests { } } - // Empty application commitments have no shard reconstruction event. - // They must become intrinsically data-available from their exact - // canonical header or the certified projection stalls behind them. let applications = (0..N as AuthorityIndex) - .map(|authority| direct_header(authority, 1, 0)) + .map(|authority| direct_header(authority, 1, 0x70 + authority as u8)) .collect::>(); let expected = applications .iter() @@ -2906,8 +2790,6 @@ mod tests { let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; let mut sync_requests = 0; - let mut projected_vertices = 0; - let mut projected_decisions = 0; pump_autonomous_until_round( &handles, &mut events, @@ -2915,8 +2797,6 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, 5, ) .await; @@ -2932,10 +2812,6 @@ mod tests { "application-critical phase carriers must not wait for a heartbeat tick" ); assert_eq!(sync_requests, 0); - assert!( - projected_vertices >= N, - "empty embedded applications must not stall clean projection" - ); drop(events); for handle in &handles { @@ -3089,8 +2965,6 @@ mod tests { let mut deliveries = vec![0; N]; let mut application_deliveries = vec![BTreeSet::new(); N]; let mut sync_requests = 0; - let mut projected_vertices = 0; - let mut projected_decisions = 0; for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); } @@ -3101,8 +2975,6 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, 2, ) .await; @@ -3154,8 +3026,6 @@ mod tests { &mut deliveries, &mut application_deliveries, &mut sync_requests, - &mut projected_vertices, - &mut projected_decisions, 10, ) .await; @@ -3748,7 +3618,6 @@ mod tests { input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), }; for round in 0..input_capacity { @@ -3799,7 +3668,6 @@ mod tests { input_capacity: 1, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::new(Mutex::new(None)), }; assert!(matches!( @@ -3834,7 +3702,6 @@ mod tests { input_capacity, desired_topology: Arc::new(Mutex::new(BTreeMap::new())), desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - desired_available_applications: Arc::new(Mutex::new(BTreeSet::new())), invalidated_by_overload: Arc::clone(&invalidated), }; for peer in 1..LARGE_N { diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 49728a65..9f5a68d0 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -409,14 +409,7 @@ impl Syncer { .with_label_values(&["local", "dropped"]) .inc(); tracing::warn!( - "Failed to enqueue RBC-DAG application carrier; the research run is invalid: {error}" - ); - } else if let Err(error) = - shadow.application_data_available(canonical.reference()) - { - self.metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - tracing::warn!( - "Failed to record local RBC-DAG application availability: {error}" + "Failed to enqueue non-authoritative RBC-DAG shadow carrier; comparison is invalid: {error}" ); } } diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 7f8f02d4..0acf9f1d 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -608,7 +608,7 @@ mod smoke_tests { } if autonomous_clock { - let autonomous_progress = tokio::time::timeout(timeout, async { + tokio::time::timeout(timeout, async { loop { if validators.iter().all(|validator| { let metrics = validator.metrics(); @@ -628,12 +628,6 @@ mod smoke_tests { .with_label_values(&["delivery", "shadow"]) .get() > 0 - && metrics.starfish_rbc_dag_projected_vertices_total.get() > 0 - && metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get() - > 0 && (!embedded_rbc_authority || metrics .starfish_rbc_dag_shadow_inputs_total @@ -647,46 +641,13 @@ mod smoke_tests { time::sleep(Duration::from_millis(25)).await; } }) - .await; - if autonomous_progress.is_err() { - let state = validators - .iter() - .map(|validator| { - let metrics = validator.metrics(); - ( - metrics.starfish_rbc_dag_shadow_clock_valid.get(), - metrics.starfish_rbc_dag_shadow_carrier_round.get(), - metrics.starfish_rbc_dag_projected_vertices_total.get(), - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get(), - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get(), - metrics.starfish_rbc_dag_shadow_pending_recovery.get(), - ) - }) - .collect::>(); - panic!( - "autonomous carrier projection did not advance while direct RBC committed; \ - per-node (valid, round, projected, commits, deliveries, recovery)={state:?}" - ); - } + .await + .expect("autonomous carrier clock did not advance while direct RBC committed"); for validator in &validators { let metrics = validator.metrics(); assert_eq!(metrics.starfish_rbc_dag_shadow_clock_valid.get(), 1); assert!(metrics.starfish_rbc_dag_shadow_carrier_round.get() > 3); - assert!(metrics.starfish_rbc_dag_projected_vertices_total.get() > 0); - assert!( - metrics - .starfish_rbc_dag_projection_decisions_total - .with_label_values(&["direct_commit"]) - .get() - > 0 - ); assert_eq!( metrics.starfish_rbc_dag_shadow_comparison_valid.get(), 0, diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 2f87d35a..236357c4 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -72,12 +72,12 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Enable the persisted RBC-DAG research runtime alongside - /// `starfish-rbc` (comparison-only unless autonomous mode is enabled). + /// Run the persisted, non-authoritative RBC-DAG shadow alongside + /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Run the independent Starfish-RBC-DAG carrier clock and certified - /// projection. Requires `--starfish-rbc-dag-shadow`. + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, /// Let embedded carrier ECHO/READY delivery certify application @@ -117,12 +117,12 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Enable the persisted RBC-DAG research runtime alongside - /// `starfish-rbc` (comparison-only unless autonomous mode is enabled). + /// Run the persisted, non-authoritative RBC-DAG shadow alongside + /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Run the independent Starfish-RBC-DAG carrier clock and certified - /// projection. Requires `--starfish-rbc-dag-shadow`. + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, /// Let embedded carrier ECHO/READY delivery certify application @@ -184,12 +184,12 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Enable the persisted RBC-DAG research runtime alongside - /// `starfish-rbc` (comparison-only unless autonomous mode is enabled). + /// Run the persisted, non-authoritative RBC-DAG shadow alongside + /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Run the independent Starfish-RBC-DAG carrier clock and certified - /// projection. Requires `--starfish-rbc-dag-shadow`. + /// Let the non-authoritative Starfish-RBC-DAG shadow create its own + /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, /// Let embedded carrier ECHO/READY delivery certify application diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 2e6aa3bd..3a01c8fb 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -17,11 +17,9 @@ implemented. The staged prototype runs under `starfish-rbc`: direct-header compa only application-header certification authority with `--starfish-rbc-dag-embedded-rbc-authority`. Direct INIT still transports the application payload, but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add -`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Autonomous -carriers now create durably locked logical consensus vertices and the clean projection produces -Starfish commit/skip decisions. Deterministic frontier output still uses the existing Starfish DAG -until milestone seven. The eventual protocol is new, not a transport option or a version-two alias -for `starfish-rbc`. +`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Consensus +projection, commit, and output still use the existing Starfish DAG. The eventual protocol is new, +not a transport option or a version-two alias for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -61,15 +59,13 @@ containing the exact canonical application header, durable application-origin re immediate application/phase scheduling, and an opt-in authority boundary that prevents direct ECHO/READY from certifying a header. Idle control heartbeats use the same resolved leader timeout as the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application carriers and their -encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. Milestone six -adds independently numbered consensus vertices, quorum strong parents, objective Vote/NoVote -choices, exact contiguous delivery frontiers, durable local consensus locks, and a live committer -that consumes only RBC-delivered, data-available projected vertices. +encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. -The current authoritative mode changes header certification and produces certified carrier-DAG -leader decisions. Direct INIT is still the application-payload transport and is not a certification -vote. The legacy Starfish DAG remains only as the temporary application-output scaffold; replacing -its commit/output path with deterministic committed frontier deltas is milestone seven. +The current authoritative mode changes only header certification. Consensus vertices, certified +projection, commits, and application ordering remain on the existing Starfish DAG. Direct INIT is +still the application-payload transport and is not a certification vote. Replacing that remaining +wrapper with a payload-only path and moving consensus into the clean carrier projection are later +milestones. Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode requires an identical recovered direct-header history, control-only autonomous history reopens @@ -931,12 +927,11 @@ Starfish-RBC; application and encodable phase carriers are immediate. The harnes generator warmup, snapshots cumulative counters at the active boundary, and drains final latency samples. -| Profile | Verdict | TPS | Block latency | E2E latency | Outbound | +| Profile | Verdict | TPS | p50 block | p50 E2E | Outbound | |---|---:|---:|---:|---:|---:| | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | | Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | | Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | -| Certified projection (milestone six) | VALID 10/10 | 799.07 | 5,102.9 ms | 8,650.9 ms | 0.50 MB/s | | Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted @@ -951,15 +946,9 @@ rounds 458–459, and ended with zero pending recovery. Its direct ECHO and READ the composed four-validator test also asserts zero such outbound messages. Its higher latency is not evidence for a bad 250 ms timer—the independent timer was removed, and the same Starfish timeout was used. It exposes the transitional clean-predecessor gate: the existing direct Starfish -DAG still serializes proposal creation on embedded delivery. - -The milestone-six run reached carrier rounds 356–359 with 35,506 carrier deliveries, 8,059 -application deliveries, 8,487 clean projected vertices, 830 clean direct commits, and zero pending -recovery. It validates the certified-projection structure, not final performance. The logical -committer currently runs beside the legacy clean-predecessor/output path, so this transitional run -pays for both and its 5.1/8.7-second latency is a red flag rather than a protocol target. Milestone -seven must make committed frontier deltas the sole ordering/output path before latency is compared -as the complete RBC-DAG protocol. +DAG still serializes proposal creation on embedded delivery. Milestone six must replace that gate +with the optimistic carrier clock plus certified consensus projection before the complete protocol +can be expected to recover Starfish pipelining. ## 19. Contained implementation milestones @@ -986,10 +975,8 @@ Every milestone is committed separately. in version-two carriers, durably reconcile their origins, schedule application/phase carriers immediately, and remove direct ECHO/READY/delivery authority. Direct INIT remains payload transport; composed tests assert zero direct ECHO/READY traffic and positive embedded delivery. -6. **Certified consensus projection (implemented):** add optional independently numbered consensus - vertices, quorum strong parents, explicit timeout-bound leader choices, contiguous exact - delivery frontiers, durable slot/choice locks, and a live clean-only direct committer. Malformed - optional vertices do not poison their enclosing carrier. +6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit + leader choice, contiguous delivery frontiers, and strict clean-only committer consumers. 7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct prefixes and anchors, and add late-node and crash/restart tests. 8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, From 1a63b52a19b28eca7e626a9b1a8176085c7907b8 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:43:18 +0200 Subject: [PATCH 56/62] Revert "Make carrier DAG RBC authoritative" This reverts commit 57d660a7506a3c37291663db789f75cb1d333bbb. --- README.md | 72 +-- crates/orchestrator/src/main.rs | 20 +- crates/orchestrator/src/protocol/starfish.rs | 13 +- crates/starfish-core/src/config.rs | 26 +- crates/starfish-core/src/metrics.rs | 99 +--- crates/starfish-core/src/net_sync.rs | 60 +-- crates/starfish-core/src/starfish_rbc.rs | 4 +- .../src/starfish_rbc_dag/journal.rs | 1 - .../starfish-core/src/starfish_rbc_dag/mod.rs | 213 +-------- .../src/starfish_rbc_dag/model.rs | 1 - .../src/starfish_rbc_dag/projection.rs | 1 - .../src/starfish_rbc_dag_shadow.rs | 100 +--- .../src/starfish_rbc_dag_shadow_service.rs | 439 ++---------------- .../starfish-core/src/starfish_rbc_service.rs | 73 +-- crates/starfish-core/src/validator.rs | 102 ++-- crates/starfish/src/main.rs | 57 +-- docs/starfish-rbc-dag-protocol.md | 81 ++-- 17 files changed, 253 insertions(+), 1109 deletions(-) diff --git a/README.md b/README.md index 30f59c49..a0fdb39f 100644 --- a/README.md +++ b/README.md @@ -50,23 +50,25 @@ Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctnes prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). **Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate logical projection. Its -canonical types, deterministic models, crash journal, comparison shadow, autonomous carrier clock, -and opt-in authoritative embedded-RBC path are implemented. Run the comparison shadow with -`--consensus starfish-rbc --starfish-rbc-dag-shadow`; add -`--starfish-rbc-dag-autonomous-clock --starfish-rbc-dag-embedded-rbc-authority` to encode exact -application headers in version-two carriers and make embedded ECHO/READY/delivery their sole -certification authority. Direct INIT remains payload transport, but direct ECHO/READY cannot clean -blocks in that mode. Idle carrier heartbeats reuse Starfish's resolved leader timeout (600 ms for -Starfish-RBC by default); application and encodable phase carriers are emitted immediately. - -The current milestone changes certification, not consensus: the existing Starfish DAG still -consumes the embedded deliveries and retains its clean-predecessor proposal gate. The certified -carrier projection and frontier linearizer are the next milestones. Shadow traffic shares the -validator's network socket and bandwidth, and deployment requires a homogeneous new-binary -committee. The default WAL is crash-safe but too intrusive for a fair latency experiment; -`--starfish-rbc-dag-shadow-buffered-wal` preserves the ordered log while syncing only on clean -shutdown and therefore forfeits crash safety. Full validator crash recovery also remains out of -scope. See the [protocol design](docs/starfish-rbc-dag-protocol.md). +canonical types, deterministic models, crash journal, direct-header comparison shadow, and a +separate opt-in autonomous heartbeat carrier clock are implemented. Run the comparison shadow with +`--consensus starfish-rbc --starfish-rbc-dag-shadow`. Add +`--starfish-rbc-dag-autonomous-clock` to run an independent control-only carrier clock (prototype +heartbeat default: 250 ms); that mode deliberately does not map direct application headers to +carrier rounds or claim a direct-delivery comparison. Direct Starfish-RBC remains solely +authoritative in both modes, and shadow failures or results cannot affect proposals, commits, or +output as protocol state. Shadow traffic still shares the validator's network socket and bandwidth, +so it can perturb timing, and it must be enabled only on a homogeneous new-binary committee; there +is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` selector is not +implemented yet. The default shadow uses per-transition fsync and a clone-based reference reducer, +so it is a correctness instrument, not a fair performance baseline, and carries no safety or +liveness claim. Benchmark runs may add `--starfish-rbc-dag-shadow-buffered-wal`; this preserves the +ordered checksummed log and syncs it on clean shutdown, but deliberately gives up crash safety for +that run. Appended and durably synchronized WAL records are reported separately. +Its WAL can reopen the shadow actor, but this is not full validator crash recovery: authoritative +direct Starfish-RBC phase and delivery locks are not durable yet, so that baseline remains fail-stop +across process restart. The design and proof obligations are documented in the +[protocol design](docs/starfish-rbc-dag-protocol.md). For a direct-header shadow comparison, `starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded observational path was disabled or shed work and the comparison must be discarded. Healthy live @@ -77,25 +79,27 @@ the cumulative direct and shadow delivery counters. Autonomous runs instead requ in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 in autonomous mode. -A matched 10-validator, 60-second-active-window local run on 2026-08-11 used the AWS RTT emulator, -nominal 1,000 tx/s load, MAC authentication, the buffered benchmark WAL, and Starfish's shared -600 ms leader/idle-carrier timeout. +A matched 10-validator, 60-second-active-window local A/B on 2026-08-11 used the AWS RTT emulator, +nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. | Profile | Verdict | TPS | Block latency | E2E latency | Outbound BW | |---|---:|---:|---:|---:|---:| | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | -| Autonomous comparison, direct RBC authoritative | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | -| Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | - -The milestone-five run produced 10,722 embedded application deliveries, reached carrier rounds -458–459, and ended with zero pending recovery. It also proves that the earlier 250 ms experimental -heartbeat was not the latency cause: application and phase carriers are already event-driven, and -using the shared 600 ms timeout did not restore the direct baseline. The remaining slowdown is an -expected warning about the transitional architecture—the old direct DAG still serializes proposal -creation on embedded RBC cleanliness. Do not tune that obsolete gate; milestone six must let the -optimistic carrier clock advance independently and feed only certified vertices into consensus. -The local harness starts its timer after transaction-generator warmup, subtracts warmup counters, -and drains the final latency samples. +| Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | + +Every shadow validator reached carrier round 275 with zero skew and pending recovery; the run +recorded 2,749 heartbeats, 27,180 embedded-RBC deliveries, and 27,424 WAL batches. Thus this +prototype's carrier/RBC work had no measurable latency penalty in this run; the extra shadow +traffic cost about 0.05 MB/s outbound. The result is not yet application latency through RBC-DAG +because direct Starfish-RBC remains authoritative. + +The same 250 ms experiment with the default crash-safe, per-transition-fsync WAL shed shadow work +and ended `INVALID` (9/10 valid validators); its authoritative path slowed to 2,569.1 ms block and +3,067.4 ms end-to-end latency. This isolates synchronous shadow persistence as the prior observer +effect, rather than a carrier-DAG latency regression. Use the buffered profile for protocol +benchmarks and the default profile for crash/replay tests; do not cite an `INVALID` run as a +protocol result. The local harness now starts its timer after transaction-generator warmup, +subtracts warmup counters, and drains the final latency samples. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's @@ -303,14 +307,14 @@ cargo run --release --bin starfish -- local-benchmark \ Additional flags: `--dissemination-mode`, `--adversarial-latency`, `--uniform-latency-ms`. -For the authoritative embedded-RBC benchmark profile: +For the non-authoritative autonomous RBC-DAG benchmark profile: ```bash cargo run --release --bin starfish -- local-benchmark \ --committee-size 10 --load 1000 --consensus starfish-rbc \ --block-authentication mac --mimic-extra-latency \ --starfish-rbc-dag-shadow --starfish-rbc-dag-autonomous-clock \ - --starfish-rbc-dag-embedded-rbc-authority \ + --starfish-rbc-dag-heartbeat-interval-ms 250 \ --starfish-rbc-dag-shadow-buffered-wal --duration-secs 60 ``` diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 761eb399..b00903d9 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -73,10 +73,9 @@ pub struct Opts { #[clap(long, global = true)] starfish_rbc_dag_autonomous_clock: bool, - /// Let embedded carrier ECHO/READY delivery certify Starfish-RBC - /// application headers. Requires autonomous RBC-DAG mode. - #[clap(long, global = true)] - starfish_rbc_dag_embedded_rbc_authority: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT", global = true)] + starfish_rbc_dag_heartbeat_interval_ms: Option, /// Benchmark-only: sync the framed shadow WAL only at clean shutdown. /// This removes per-transition disk pressure and is not crash-safe. @@ -92,7 +91,7 @@ pub struct Opts { struct StarfishRbcDagOverrides { shadow: bool, autonomous_clock: bool, - embedded_rbc_authority: bool, + heartbeat_interval_ms: Option, buffered_wal: bool, } @@ -897,8 +896,8 @@ fn load_benchmark_configs( if starfish_rbc_dag.autonomous_clock { node_parameters.starfish_rbc_dag_autonomous_clock = true; } - if starfish_rbc_dag.embedded_rbc_authority { - node_parameters.starfish_rbc_dag_embedded_rbc_authority = true; + if let Some(interval_ms) = starfish_rbc_dag.heartbeat_interval_ms { + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; } if starfish_rbc_dag.buffered_wal { node_parameters.starfish_rbc_dag_shadow_buffered_wal = true; @@ -1086,7 +1085,7 @@ async fn run( let starfish_rbc_dag = StarfishRbcDagOverrides { shadow: opts.starfish_rbc_dag_shadow, autonomous_clock: opts.starfish_rbc_dag_autonomous_clock, - embedded_rbc_authority: opts.starfish_rbc_dag_embedded_rbc_authority, + heartbeat_interval_ms: opts.starfish_rbc_dag_heartbeat_interval_ms, buffered_wal: opts.starfish_rbc_dag_shadow_buffered_wal, }; match opts.operation { @@ -2384,7 +2383,8 @@ mod tests { "mac", "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", - "--starfish-rbc-dag-embedded-rbc-authority", + "--starfish-rbc-dag-heartbeat-interval-ms", + "125", "--starfish-rbc-dag-shadow-buffered-wal", "--protocols", "starfish-rbc", @@ -2394,8 +2394,8 @@ mod tests { assert_eq!(opts.block_authentication.as_deref(), Some("mac")); assert!(opts.starfish_rbc_dag_shadow); assert!(opts.starfish_rbc_dag_autonomous_clock); - assert!(opts.starfish_rbc_dag_embedded_rbc_authority); assert!(opts.starfish_rbc_dag_shadow_buffered_wal); + assert_eq!(opts.starfish_rbc_dag_heartbeat_interval_ms, Some(125)); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); }; diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 447b6e56..1bb38995 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -324,8 +324,9 @@ impl StarfishProtocol { // validator configuration. node_parameters.starfish_rbc_dag_shadow = false; node_parameters.starfish_rbc_dag_autonomous_clock = false; - node_parameters.starfish_rbc_dag_embedded_rbc_authority = false; node_parameters.starfish_rbc_dag_shadow_buffered_wal = false; + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = + config::node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(); } node_parameters } @@ -362,13 +363,14 @@ impl StarfishProtocol { mod tests { use super::{ProtocolMetrics, StarfishNodeParameters, StarfishProtocol}; use crate::{benchmark::BenchmarkParameters, client::Instance}; - use starfish_core::config::NodeParameters; + use starfish_core::config::{NodeParameters, node_defaults}; #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); @@ -382,6 +384,7 @@ mod tests { assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); assert!(parameters.starfish_rbc_dag_shadow_buffered_wal); + assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } #[test] @@ -389,6 +392,7 @@ mod tests { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); @@ -407,6 +411,11 @@ mod tests { !parameters.starfish_rbc_dag_shadow_buffered_wal, "a global buffered-WAL flag must not leak into non-RBC comparison members" ); + assert_eq!( + parameters.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), + "a global heartbeat override must not leak into non-RBC comparison members" + ); } #[test] diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index c43c27cf..cf419614 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -77,11 +77,6 @@ pub struct NodeParameters { /// `starfish_rbc_dag_shadow`. #[serde(default)] pub starfish_rbc_dag_autonomous_clock: bool, - /// Use embedded carrier ECHO/READY delivery as the certification authority - /// for Starfish-RBC application headers. Direct RBC retains INIT/payload - /// transport but its phase messages cannot mark a block clean. - #[serde(default)] - pub starfish_rbc_dag_embedded_rbc_authority: bool, /// Testbed-only receiver-local single-DAG RBC path: deliver an exact header /// after locally observing quorum ECHO rather than quorum READY. Quorum /// intersection preserves a unique value, but pairwise-MAC testimony is @@ -145,6 +140,10 @@ pub mod node_defaults { 5 } + pub fn default_starfish_rbc_dag_heartbeat_interval_ms() -> u64 { + 250 + } + pub fn default_causal_push_shard_round_lag() -> RoundNumber { 0 } @@ -172,7 +171,6 @@ impl Default for NodeParameters { starfish_rbc_protocol_instance: None, starfish_rbc_dag_shadow: false, starfish_rbc_dag_autonomous_clock: false, - starfish_rbc_dag_embedded_rbc_authority: false, starfish_rbc_single_dag_echo_qc_fast_path: false, starfish_rbc_dag_shadow_buffered_wal: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), @@ -433,9 +431,9 @@ impl ImportExport for NodePrivateConfig {} #[cfg(test)] mod tests { - use std::{path::Path, time::Duration}; + use std::path::Path; - use super::{NodeParameters, NodePrivateConfig}; + use super::{NodeParameters, NodePrivateConfig, node_defaults}; #[test] fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { @@ -445,6 +443,10 @@ mod tests { assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); + assert_eq!( + parameters.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() + ); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); assert_ne!(protocol_instance, [0; 32]); @@ -459,6 +461,10 @@ mod tests { assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); + assert_eq!( + decoded.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() + ); } #[test] @@ -466,7 +472,7 @@ mod tests { let parameters = NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, - leader_timeout: Duration::from_millis(125), + starfish_rbc_dag_heartbeat_interval_ms: 125, starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -476,7 +482,7 @@ mod tests { assert!(decoded.starfish_rbc_dag_shadow); assert!(decoded.starfish_rbc_dag_autonomous_clock); assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); - assert_eq!(decoded.leader_timeout, Duration::from_millis(125)); + assert_eq!(decoded.starfish_rbc_dag_heartbeat_interval_ms, 125); } #[test] diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 5d47fc66..a00075c0 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -267,7 +267,6 @@ pub struct MetricReporter { pub struct AutonomousClockBenchmarkBaseline { accepted_heartbeats: u64, delivered_carriers: u64, - delivered_applications: u64, wal_batches: u64, wal_records: u64, carrier_round: i64, @@ -292,7 +291,6 @@ struct AutonomousClockBenchmarkSummary { bounded_nodes: usize, accepted_heartbeats: u64, delivered_carriers: u64, - delivered_applications: u64, wal_batches: u64, wal_records: u64, pending_recovery: i64, @@ -311,7 +309,6 @@ fn summarize_autonomous_clock_benchmark( metrics: &[Arc], committee_size: usize, baselines: Option<&[AutonomousClockBenchmarkBaseline]>, - embedded_rbc_authority: bool, ) -> AutonomousClockBenchmarkSummary { let committee_size = i64::try_from(committee_size).unwrap_or(i64::MAX); let maximum_phase_backlog_bound = @@ -341,12 +338,6 @@ fn summarize_autonomous_clock_benchmark( .with_label_values(&["delivery", "shadow"]) .get() > baseline.delivered_carriers - && (!embedded_rbc_authority - || metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .get() - > baseline.delivered_applications) && metrics .starfish_rbc_dag_shadow_wal_appended_batches_total .get() @@ -394,15 +385,6 @@ fn summarize_autonomous_clock_benchmark( .get() }) .sum(); - let delivered_applications = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .get() - }) - .sum(); let wal_batches = metrics .iter() .map(|metrics| { @@ -469,7 +451,6 @@ fn summarize_autonomous_clock_benchmark( bounded_nodes, accepted_heartbeats, delivered_carriers, - delivered_applications, wal_batches, wal_records, pending_recovery, @@ -506,10 +487,6 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .get(), - delivered_applications: self - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .get(), wal_batches: self .starfish_rbc_dag_shadow_wal_appended_batches_total .get(), @@ -1429,7 +1406,6 @@ impl Metrics { committee_size: usize, starfish_rbc_dag_shadow_expected: bool, starfish_rbc_dag_autonomous_clock_expected: bool, - starfish_rbc_dag_embedded_rbc_authority_expected: bool, autonomous_clock_baselines: Option>, counter_baselines: Option>, ) { @@ -1677,18 +1653,11 @@ impl Metrics { &metrics, committee_size, autonomous_clock_baselines.as_deref(), - starfish_rbc_dag_embedded_rbc_authority_expected, ); let round_lag = summary.maximum_round.saturating_sub(summary.minimum_round); table.add_row(row![bH2->""]); - table.add_row(row![ - bH2->if starfish_rbc_dag_embedded_rbc_authority_expected { - "RBC-DAG Embedded RBC Authority Verification" - } else { - "RBC-DAG Autonomous Clock Verification" - } - ]); + table.add_row(row![bH2->"RBC-DAG Autonomous Clock Verification"]); table.add_row(row![ b->"Clock verdict:", if summary.verdict_valid { @@ -1712,10 +1681,9 @@ impl Metrics { table.add_row(row![ b->"Clock/WAL progress:", format!( - "heartbeats={}, carrier deliveries={}, application deliveries={}, WAL batches={}, records={}, open rounds={}..{}", + "heartbeats={}, RBC deliveries={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, summary.delivered_carriers, - summary.delivered_applications, summary.wal_batches, summary.wal_records, summary.minimum_round, @@ -2311,7 +2279,7 @@ mod tests { autonomous_clock_metrics(11, 6, 1), ]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); assert!(summary.verdict_valid); assert_eq!(summary.valid_nodes, 4); @@ -2336,10 +2304,7 @@ mod tests { .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) .collect::>(); - assert!( - !summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) - .verdict_valid - ); + assert!(!summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); for metrics in &metrics { metrics @@ -2365,59 +2330,7 @@ mod tests { metrics.starfish_rbc_dag_shadow_carrier_round.inc(); } - assert!( - summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines), false) - .verdict_valid - ); - } - - #[test] - fn embedded_authority_summary_requires_application_delivery_progress() { - let metrics = vec![autonomous_clock_metrics(8, 0, 0)]; - let baselines = metrics - .iter() - .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) - .collect::>(); - let metrics = &metrics[0]; - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_records_total - .inc(); - metrics.starfish_rbc_dag_shadow_carrier_round.inc(); - - assert!( - !summarize_autonomous_clock_benchmark( - &[Arc::clone(metrics)], - 2, - Some(&baselines), - true, - ) - .verdict_valid - ); - - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .inc(); - assert!( - summarize_autonomous_clock_benchmark( - &[Arc::clone(metrics)], - 2, - Some(&baselines), - true, - ) - .verdict_valid - ); + assert!(summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); } #[test] @@ -2436,7 +2349,7 @@ mod tests { ); let metrics = vec![no_progress, invalid_clock, unbounded]; - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None, false); + let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); assert!(!summary.verdict_valid); assert_eq!(summary.valid_nodes, 2); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index c400fb5b..dc3daaf8 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -45,7 +45,7 @@ use crate::{ SailfishCertEvent, SailfishServiceHandle, SailfishServiceMessage, start_sailfish_service, }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, - starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader, RbcCommitteeId, RbcProtocolInstanceId}, + starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, starfish_rbc_dag::{ RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, storage::ShadowWalSyncPolicyV1, @@ -59,8 +59,7 @@ use crate::{ start_starfish_rbc_dag_shadow_service_v1, }, starfish_rbc_service::{ - RbcInitialAuthenticator, RbcPhaseAuthorityV1, RbcServiceEvent, RbcServiceHandle, - start_starfish_rbc_service_with_phase_authority, + RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, }, syncer::{CommitObserver, STARFISH_RBC_SINGLE_DAG_ROUND_INTERVAL, Syncer, SyncerSignals}, types::{ @@ -1811,7 +1810,13 @@ impl NetworkSyncer let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); - let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { + let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow + && node_parameters.starfish_rbc_dag_autonomous_clock + { + // Autonomous carrier rounds are independent of direct consensus + // rounds and recover entirely from their distinct WAL. + Some(Vec::new()) + } else if node_parameters.starfish_rbc_dag_shadow { match recovered_local_rbc_headers(&core) { Ok(headers) => Some(headers), Err(error) => { @@ -1951,11 +1956,9 @@ impl NetworkSyncer dag_state.get_own_authority_index(), context, authorizer, - recovered_local_headers, - // The idle carrier pacemaker deliberately shares the - // resolved Starfish leader timeout. Application and - // embedded RBC phase carriers remain event-driven. - node_parameters.leader_timeout, + Duration::from_millis( + node_parameters.starfish_rbc_dag_heartbeat_interval_ms, + ), wal_sync_policy, ) } else { @@ -2211,11 +2214,6 @@ impl NetworkSyncer }) }); - let embedded_rbc_authority = node_parameters.starfish_rbc_dag_embedded_rbc_authority; - let embedded_rbc_committee_id = embedded_rbc_authority.then(|| { - RbcCommitteeId::derive(&inner.committee) - .expect("validated direct RBC committee must retain a stable identifier") - }); let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); let shadow_metrics = metrics.clone(); @@ -2271,40 +2269,6 @@ impl NetworkSyncer .inc(); tracing::debug!(?identity, "RBC-DAG shadow delivered carrier"); } - ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header } => { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .inc(); - tracing::debug!( - ?carrier, - application = ?header.reference(), - "RBC-DAG shadow delivered embedded application header" - ); - if embedded_rbc_authority { - match PinnedRbcHeader::validate_with_committee_id( - header, - &event_inner.committee, - embedded_rbc_committee_id - .expect("embedded authority must cache its committee ID"), - ) { - Ok(header) => { - event_inner - .syncer - .apply_starfish_rbc_deliveries(vec![header]) - .await; - } - Err(error) => { - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - tracing::error!( - ?carrier, - ?error, - "Embedded RBC delivered an invalid application header" - ); - } - } - } - } ShadowServiceEventV1::ComparisonBacklog { unpaired_direct, unpaired_shadow, diff --git a/crates/starfish-core/src/starfish_rbc.rs b/crates/starfish-core/src/starfish_rbc.rs index 4096a982..cc78f5c5 100644 --- a/crates/starfish-core/src/starfish_rbc.rs +++ b/crates/starfish-core/src/starfish_rbc.rs @@ -539,7 +539,7 @@ pub(crate) struct PinnedRbcHeader { } impl PinnedRbcHeader { - pub(crate) fn validate_with_committee_id( + fn validate_with_committee_id( header: RbcCanonicalHeader, committee: &Committee, committee_id: RbcCommitteeId, @@ -745,7 +745,7 @@ impl fmt::Debug for RbcProtocolInstanceId { pub(crate) struct RbcCommitteeId([u8; COMMITTEE_ID_SIZE]); impl RbcCommitteeId { - pub(crate) fn derive(committee: &Committee) -> Result { + fn derive(committee: &Committee) -> Result { if committee.len() > MAX_COMMITTEE_SIZE as usize { return Err(RbcError::CommitteeTooLarge(committee.len())); } diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index bdb4fb1d..24812f40 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -1229,7 +1229,6 @@ mod tests { own_prev: parent(author), weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - application_header: None, data_acknowledgments: Vec::new(), phase_batch, consensus_vertex, diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs index f734e527..31f82af0 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -27,7 +27,6 @@ use crate::{ MacTag, MlDsa44SignatureBytes, MlDsa44Signer, MlDsa65SignatureBytes, MlDsa65Signer, SIGNATURE_SIZE, SignatureBytes, Signer, TransactionsCommitment, }, - starfish_rbc::RbcCanonicalHeader, types::{ AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, RoundNumber, TimestampNs, @@ -36,10 +35,6 @@ use crate::{ pub const CARRIER_FORMAT_VERSION_V1: u8 = 1; pub const CARRIER_WIRE_FORMAT_VERSION_V1: u8 = 0x81; -/// V2 extends V1 with one complete canonical application header. Control-only -/// carriers remain byte-for-byte V1 so existing autonomous-clock WALs reopen. -pub const CARRIER_FORMAT_VERSION_V2: u8 = 2; -pub const CARRIER_WIRE_FORMAT_VERSION_V2: u8 = 0x82; pub const MAX_CARRIER_CONTENT_SIZE_V1: usize = 4 * 1024 * 1024; pub const MAX_PHASE_STATEMENTS_V1: usize = 2_048; @@ -53,7 +48,6 @@ const ACKNOWLEDGMENTS_FIELD: u8 = 0x06; const PHASE_BATCH_FIELD: u8 = 0x07; const CONSENSUS_VERTEX_FIELD: u8 = 0x08; const CREATION_TIME_FIELD: u8 = 0x09; -const APPLICATION_HEADER_FIELD: u8 = 0x0A; const CONSENSUS_ROUND_FIELD: u8 = 0x01; const STRONG_PARENTS_FIELD: u8 = 0x02; const DELIVERY_FRONTIER_FIELD: u8 = 0x03; @@ -323,7 +317,6 @@ pub struct CarrierHeaderV1 { own_prev: BlockReference, weak_parents: Vec, transactions_commitment: TransactionsCommitment, - application_header: Option, data_acknowledgments: Vec, phase_batch: Vec, consensus_vertex: Option, @@ -337,9 +330,6 @@ pub struct CarrierHeaderV1Args { pub own_prev: BlockReference, pub weak_parents: Vec, pub transactions_commitment: TransactionsCommitment, - /// Exact application header disseminated by this carrier. `None` denotes - /// a control-only heartbeat and retains the frozen V1 byte grammar. - pub application_header: Option, pub data_acknowledgments: Vec, pub phase_batch: Vec, pub consensus_vertex: Option, @@ -354,7 +344,6 @@ impl CarrierHeaderV1 { own_prev: args.own_prev, weak_parents: args.weak_parents, transactions_commitment: args.transactions_commitment, - application_header: args.application_header, data_acknowledgments: args.data_acknowledgments, phase_batch: args.phase_batch, consensus_vertex: args.consensus_vertex, @@ -384,10 +373,6 @@ impl CarrierHeaderV1 { self.transactions_commitment } - pub fn application_header(&self) -> Option<&RbcCanonicalHeader> { - self.application_header.as_ref() - } - pub fn data_acknowledgments(&self) -> &[BlockReference] { &self.data_acknowledgments } @@ -1557,12 +1542,6 @@ pub enum RbcDagError { InvalidWeakParent(BlockReference), WeakParentsNotOrdered, InvalidCarrierThreshold, - InvalidApplicationHeader, - ApplicationAuthorMismatch { - carrier: AuthorityIndex, - application: AuthorityIndex, - }, - ApplicationCommitmentMismatch, InvalidAcknowledgment(BlockReference), DuplicateAcknowledgment(BlockReference), InvalidPhaseTarget(BlockReference), @@ -1662,22 +1641,6 @@ fn validate_outer_header( return Err(RbcDagError::InvalidCarrierThreshold); } - if let Some(application_header) = &header.application_header { - application_header - .validate_for_committee(committee) - .map_err(|_| RbcDagError::InvalidApplicationHeader)?; - let application = application_header.reference().authority; - if application != header.author { - return Err(RbcDagError::ApplicationAuthorMismatch { - carrier: header.author, - application, - }); - } - if application_header.transactions_commitment() != header.transactions_commitment { - return Err(RbcDagError::ApplicationCommitmentMismatch); - } - } - if header.data_acknowledgments.len() > u16::MAX as usize { return Err(RbcDagError::VectorTooLong { field: "acknowledgments", @@ -1773,14 +1736,10 @@ fn encode_header( ) -> Result, RbcDagError> { let mut bytes = Vec::new(); bytes.push(CONTENT_FORMAT_FIELD); - bytes.push( - match (acknowledgment_encoding, header.application_header.is_some()) { - (AckEncoding::Expanded, false) => CARRIER_FORMAT_VERSION_V1, - (AckEncoding::Compressed, false) => CARRIER_WIRE_FORMAT_VERSION_V1, - (AckEncoding::Expanded, true) => CARRIER_FORMAT_VERSION_V2, - (AckEncoding::Compressed, true) => CARRIER_WIRE_FORMAT_VERSION_V2, - }, - ); + bytes.push(match acknowledgment_encoding { + AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, + AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, + }); bytes.push(AUTHOR_FIELD); bytes.extend_from_slice(&header.author.to_be_bytes()); bytes.push(CARRIER_ROUND_FIELD); @@ -1794,10 +1753,6 @@ fn encode_header( } bytes.push(TRANSACTIONS_COMMITMENT_FIELD); bytes.extend_from_slice(header.transactions_commitment.as_ref()); - if let Some(application_header) = &header.application_header { - bytes.push(APPLICATION_HEADER_FIELD); - encode_application_header(&mut bytes, application_header)?; - } bytes.push(ACKNOWLEDGMENTS_FIELD); match acknowledgment_encoding { AckEncoding::Expanded => { @@ -1841,37 +1796,6 @@ fn encode_header( Ok(bytes) } -fn encode_application_header( - bytes: &mut Vec, - header: &RbcCanonicalHeader, -) -> Result<(), RbcDagError> { - let reference = header.reference(); - bytes.push(0x01); - bytes.extend_from_slice(&reference.authority.to_be_bytes()); - bytes.push(0x02); - bytes.extend_from_slice(&reference.round.to_be_bytes()); - bytes.push(0x03); - encode_count( - bytes, - "application parents", - header.block_references().len(), - )?; - for parent in header.block_references() { - encode_reference(bytes, *parent); - } - let acknowledgments = header.acknowledgment_references(); - bytes.push(0x04); - encode_count(bytes, "application acknowledgments", acknowledgments.len())?; - for acknowledgment in acknowledgments { - encode_reference(bytes, acknowledgment); - } - bytes.push(0x05); - bytes.extend_from_slice(&header.meta_creation_time_ns().to_be_bytes()); - bytes.push(0x06); - bytes.extend_from_slice(header.transactions_commitment().as_ref()); - Ok(()) -} - fn encode_consensus_vertex( bytes: &mut Vec, vertex: &ConsensusVertexV1, @@ -1984,15 +1908,13 @@ fn decode_header( let mut decoder = Decoder::new(bytes); decoder.expect_marker(CONTENT_FORMAT_FIELD)?; let version = decoder.read_u8()?; - let has_application_header = match (acknowledgment_encoding, version) { - (AckEncoding::Expanded, CARRIER_FORMAT_VERSION_V1) - | (AckEncoding::Compressed, CARRIER_WIRE_FORMAT_VERSION_V1) => false, - (AckEncoding::Expanded, CARRIER_FORMAT_VERSION_V2) - | (AckEncoding::Compressed, CARRIER_WIRE_FORMAT_VERSION_V2) => true, - _ => { - return Err(RbcDagError::UnsupportedVersion(version)); - } + let expected_version = match acknowledgment_encoding { + AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, + AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, }; + if version != expected_version { + return Err(RbcDagError::UnsupportedVersion(version)); + } decoder.expect_marker(AUTHOR_FIELD)?; let author = decoder.read_u16()?; decoder.expect_marker(CARRIER_ROUND_FIELD)?; @@ -2004,12 +1926,6 @@ fn decode_header( let weak_parents = decoder.read_references(weak_count)?; decoder.expect_marker(TRANSACTIONS_COMMITMENT_FIELD)?; let transactions_commitment = TransactionsCommitment::from_bytes(decoder.read_array()?); - let application_header = if has_application_header { - decoder.expect_marker(APPLICATION_HEADER_FIELD)?; - Some(decoder.read_application_header()?) - } else { - None - }; decoder.expect_marker(ACKNOWLEDGMENTS_FIELD)?; let data_acknowledgments = match acknowledgment_encoding { AckEncoding::Expanded => { @@ -2034,7 +1950,6 @@ fn decode_header( own_prev, weak_parents: weak_parents.clone(), transactions_commitment, - application_header: application_header.clone(), data_acknowledgments: acknowledgments.clone(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2075,7 +1990,6 @@ fn decode_header( own_prev, weak_parents, transactions_commitment, - application_header, data_acknowledgments, phase_batch, consensus_vertex, @@ -2128,33 +2042,6 @@ impl<'a> Decoder<'a> { Ok(u64::from_be_bytes(self.read_array()?)) } - fn read_application_header(&mut self) -> Result { - self.expect_marker(0x01)?; - let author = self.read_u16()?; - self.expect_marker(0x02)?; - let round = self.read_u32()?; - self.expect_marker(0x03)?; - let parent_count = self.read_count("application parents", u16::MAX as usize)?; - let parents = self.read_references(parent_count)?; - self.expect_marker(0x04)?; - let acknowledgment_count = - self.read_count("application acknowledgments", u16::MAX as usize)?; - let acknowledgments = self.read_references(acknowledgment_count)?; - self.expect_marker(0x05)?; - let creation_time_ns = self.read_u64()?; - self.expect_marker(0x06)?; - let transactions_commitment = TransactionsCommitment::from_bytes(self.read_array()?); - RbcCanonicalHeader::try_new( - author, - round, - parents, - acknowledgments, - creation_time_ns, - transactions_commitment, - ) - .map_err(|_| RbcDagError::InvalidApplicationHeader) - } - fn expect_marker(&mut self, expected: u8) -> Result<(), RbcDagError> { let actual = self.read_u8()?; if actual != expected { @@ -2296,7 +2183,6 @@ mod tests { use crate::crypto::{ dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, }; - use crate::types::VerifiedBlock; fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { BlockReference { @@ -2342,7 +2228,6 @@ mod tests { .map(parent) .collect(), transactions_commitment: TransactionsCommitment::from_bytes([0x55; 32]), - application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2381,25 +2266,6 @@ mod tests { CandidateCarrierV1::try_new(full_args(committee), committee).unwrap() } - fn application_header( - committee: &Committee, - author: AuthorityIndex, - marker: u8, - ) -> RbcCanonicalHeader { - RbcCanonicalHeader::try_new( - author, - 1, - committee - .authorities() - .map(|authority| *VerifiedBlock::new_genesis(authority).reference()) - .collect(), - Vec::new(), - 0x1122_3344_5566_7700 + u64::from(marker), - TransactionsCommitment::from_bytes([marker; 32]), - ) - .unwrap() - } - #[test] fn canonical_content_and_reference_have_frozen_golden_bytes() { let committee = Committee::new_test(vec![1; 4]); @@ -2452,65 +2318,6 @@ mod tests { ); } - #[test] - fn application_carrier_uses_v2_and_round_trips_the_exact_canonical_header() { - let committee = Committee::new_test(vec![1; 4]); - let application = application_header(&committee, 3, 0xA5); - let mut application_args = args(&committee, 3, 2); - application_args.transactions_commitment = application.transactions_commitment(); - application_args.application_header = Some(application.clone()); - let candidate = CandidateCarrierV1::try_new(application_args.clone(), &committee).unwrap(); - let content = candidate.canonical_content_bytes().unwrap(); - let wire = candidate.canonical_wire_bytes().unwrap(); - assert_eq!(content[1], CARRIER_FORMAT_VERSION_V2); - assert_eq!(wire[1], CARRIER_WIRE_FORMAT_VERSION_V2); - - let decoded_content = - CandidateCarrierV1::decode_content(&content, &committee, Some(candidate.reference())) - .unwrap(); - let decoded_wire = - CandidateCarrierV1::decode_wire(&wire, &committee, Some(candidate.reference())) - .unwrap(); - assert_eq!( - decoded_content.header().application_header(), - Some(&application) - ); - assert_eq!(decoded_wire, decoded_content); - - let changed_application = application_header(&committee, 3, 0xA6); - application_args.transactions_commitment = changed_application.transactions_commitment(); - application_args.application_header = Some(changed_application); - assert_ne!( - CandidateCarrierV1::try_new(application_args, &committee) - .unwrap() - .reference(), - candidate.reference() - ); - } - - #[test] - fn application_carrier_rejects_author_or_commitment_mismatch() { - let committee = Committee::new_test(vec![1; 4]); - let application = application_header(&committee, 3, 0xB1); - let mut bad_commitment = args(&committee, 3, 2); - bad_commitment.application_header = Some(application.clone()); - assert_eq!( - CandidateCarrierV1::try_new(bad_commitment, &committee), - Err(RbcDagError::ApplicationCommitmentMismatch) - ); - - let mut bad_author = args(&committee, 2, 2); - bad_author.transactions_commitment = application.transactions_commitment(); - bad_author.application_header = Some(application); - assert_eq!( - CandidateCarrierV1::try_new(bad_author, &committee), - Err(RbcDagError::ApplicationAuthorMismatch { - carrier: 2, - application: 3, - }) - ); - } - #[test] fn every_canonical_carrier_field_is_bound_to_the_reference() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index 88504e35..192385aa 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -1459,7 +1459,6 @@ mod tests { own_prev, weak_parents, transactions_commitment: TransactionsCommitment::default(), - application_header: None, data_acknowledgments: Vec::new(), phase_batch, consensus_vertex: None, diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index f0a94206..1bda66fe 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -839,7 +839,6 @@ mod tests { own_prev: previous[author as usize], weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: vertex, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index a81e0d71..1fcfd07d 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -18,7 +18,6 @@ use std::{ use crate::{ crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, - starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ AuthenticatedCarrierV1, CandidateCarrierV1, CarrierAuthenticationV1, CarrierAuthorizerV1, CarrierHeaderV1Args, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, @@ -123,15 +122,6 @@ pub(crate) struct ShadowOutboundEnvelopeV1 { authentication_sidecar: Vec, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct LocalOutboundMetadataV1 { - pub(crate) round: RoundNumber, - pub(crate) transactions_commitment: TransactionsCommitment, - pub(crate) creation_time_ns: TimestampNs, - pub(crate) control_shape: bool, - pub(crate) application: Option, -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ShadowIngressDispositionV1 { Authenticated, @@ -584,18 +574,6 @@ impl StarfishRbcDagShadowV1 { self.model.pending_phase_backlog_len() } - /// Whether the exact phase prefix encodable in the open carrier contains - /// work for an application-bearing carrier. This lets the prototype send - /// application-critical ECHO/READY promptly without turning control-only - /// carrier certification into an unpaced self-sustaining loop. - pub(crate) fn has_pending_application_phase_work(&self) -> bool { - self.model.pending_phase_batch().iter().any(|statement| { - self.candidates - .get(&statement.target()) - .is_some_and(|candidate| candidate.header().application_header().is_some()) - }) - } - pub(crate) fn admitted_reference( &self, authority: AuthorityIndex, @@ -655,21 +633,6 @@ impl StarfishRbcDagShadowV1 { round: RoundNumber, transactions_commitment: TransactionsCommitment, creation_time_ns: TimestampNs, - ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { - self.create_local_carrier_with_application( - round, - transactions_commitment, - None, - creation_time_ns, - ) - } - - fn create_local_carrier_with_application( - &mut self, - round: RoundNumber, - transactions_commitment: TransactionsCommitment, - application_header: Option, - creation_time_ns: TimestampNs, ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { self.ensure_live()?; let (own_prev, weak_parents) = self.model.local_parent_set()?; @@ -680,7 +643,6 @@ impl StarfishRbcDagShadowV1 { own_prev, weak_parents, transactions_commitment, - application_header, data_acknowledgments: Vec::new(), phase_batch: self.model.pending_phase_batch(), consensus_vertex: None, @@ -719,24 +681,6 @@ impl StarfishRbcDagShadowV1 { self.create_local_carrier(round, TransactionsCommitment::default(), creation_time_ns) } - /// Assign one exact direct application header to the currently open - /// independent carrier slot. The complete canonical header is committed - /// by the V2 carrier and is therefore recoverable from carrier content. - pub(crate) fn create_local_application_carrier( - &mut self, - application_header: RbcCanonicalHeader, - creation_time_ns: TimestampNs, - ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { - let round = self.model.local_carrier_round(); - let commitment = application_header.transactions_commitment(); - self.create_local_carrier_with_application( - round, - commitment, - Some(application_header), - creation_time_ns, - ) - } - /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -965,7 +909,7 @@ impl StarfishRbcDagShadowV1 { /// payload and creation timestamp before accepting new observations. pub(crate) fn local_outbound_metadata( &self, - ) -> Result, ShadowErrorV1> { + ) -> Result, ShadowErrorV1> { self.journal .snapshot() .retransmissions() @@ -976,17 +920,13 @@ impl StarfishRbcDagShadowV1 { .candidates .get(&reference) .ok_or(ShadowErrorV1::MissingOutboundCandidate(reference))?; - Ok(LocalOutboundMetadataV1 { - round: candidate.header().carrier_round(), - transactions_commitment: candidate.header().transactions_commitment(), - creation_time_ns: candidate.header().creation_time_ns(), - control_shape: candidate.header().data_acknowledgments().is_empty() + Ok(( + candidate.header().carrier_round(), + candidate.header().transactions_commitment(), + candidate.header().creation_time_ns(), + candidate.header().data_acknowledgments().is_empty() && candidate.header().consensus_vertex().is_none(), - application: candidate - .header() - .application_header() - .map(RbcCanonicalHeader::reference), - }) + )) }) .collect() } @@ -1010,30 +950,6 @@ impl StarfishRbcDagShadowV1 { .collect() } - /// Exact application headers whose enclosing carriers reached embedded - /// RBC delivery. Control-only carrier deliveries are intentionally absent. - pub(crate) fn delivered_application_headers( - &self, - ) -> Result, ShadowErrorV1> { - self.delivered - .iter() - .filter_map(|carrier_reference| { - let candidate = match self.candidates.get(carrier_reference) { - Some(candidate) => candidate, - None => { - return Some(Err(ShadowErrorV1::MissingDeliveredCandidate( - *carrier_reference, - ))); - } - }; - candidate - .header() - .application_header() - .map(|header| Ok((*carrier_reference, header.clone()))) - }) - .collect() - } - /// Compare protocol-independent delivery sets. Multiple transaction /// commitments for one `(author, round)` slot make the comparison /// ambiguous instead of being resolved by arrival or reference order. @@ -2934,7 +2850,6 @@ mod tests { own_prev: carrier_genesis_reference(author), weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2971,7 +2886,6 @@ mod tests { transactions_commitment: TransactionsCommitment::from_bytes( [0xD0 + author as u8; 32], ), - application_header: None, data_acknowledgments: Vec::new(), phase_batch: vec![statement], consensus_vertex: None, diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 0f3b784d..62501be8 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -97,13 +97,12 @@ impl ShadowServiceModeV1 { } } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] struct ShadowLocalCarrierV1 { author: AuthorityIndex, round: RoundNumber, transactions_commitment: crate::crypto::TransactionsCommitment, creation_time_ns: TimestampNs, - application_header: RbcCanonicalHeader, } impl ShadowLocalCarrierV1 { @@ -113,7 +112,6 @@ impl ShadowLocalCarrierV1 { round: header.reference().round, transactions_commitment: header.transactions_commitment(), creation_time_ns: header.meta_creation_time_ns(), - application_header: header.clone(), } } } @@ -182,6 +180,12 @@ impl StarfishRbcDagShadowServiceHandleV1 { &self, header: &RbcCanonicalHeader, ) -> Result<(), ShadowServiceErrorV1> { + if self.mode.is_autonomous() { + // The autonomous carrier clock is deliberately independent from + // direct consensus rounds. Direct headers continue through the + // authoritative path and cannot consume carrier slots. + return Ok(()); + } self.send(ShadowServiceMessageV1::LocalCarrier( ShadowLocalCarrierV1::from_direct_header(header), )) @@ -368,10 +372,6 @@ pub(crate) enum ShadowServiceEventV1 { message: NetworkMessage, }, Delivered(ShadowDeliveryIdentityV1), - EmbeddedApplicationDelivered { - carrier: BlockReference, - header: RbcCanonicalHeader, - }, Comparison(ShadowDeliveryComparisonV1), Input { kind: &'static str, @@ -420,7 +420,7 @@ pub(crate) enum ShadowServiceErrorV1 { ConflictingLocalHeader(RoundNumber), MissingRecoveredLocalHeader(RoundNumber), RecoveredLocalHeaderMismatch(RoundNumber), - AutonomousWalContainsInvalidCarrier(RoundNumber), + AutonomousWalContainsApplicationCarrier(RoundNumber), LocalHeaderAuthority { expected: AuthorityIndex, actual: AuthorityIndex, @@ -501,9 +501,9 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "persisted shadow carrier and recovered direct header disagree at round {round}" ), - Self::AutonomousWalContainsInvalidCarrier(round) => write!( + Self::AutonomousWalContainsApplicationCarrier(round) => write!( formatter, - "autonomous carrier-clock WAL contains an invalid local carrier at round {round}" + "autonomous carrier-clock WAL contains a non-heartbeat local carrier at round {round}" ), Self::LocalHeaderAuthority { expected, actual } => write!( formatter, @@ -593,7 +593,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( own_authority: AuthorityIndex, context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, heartbeat_interval: Duration, wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< @@ -613,7 +612,7 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( own_authority, context, authorizer, - recovered_local_headers, + Vec::new(), ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, wal_sync_policy, ) @@ -650,10 +649,9 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( actual: local.author, }); } - let round = local.round; - if let Some(previous) = pending_local.insert(round, local.clone()) { + if let Some(previous) = pending_local.insert(local.round, local) { if previous != local { - return Err(ShadowServiceErrorV1::ConflictingLocalHeader(round)); + return Err(ShadowServiceErrorV1::ConflictingLocalHeader(local.round)); } } } @@ -761,16 +759,8 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let persisted_local = match core.local_outbound_metadata() { Ok(metadata) => metadata .into_iter() - .map(|metadata| { - ( - metadata.round, - ( - metadata.transactions_commitment, - metadata.creation_time_ns, - metadata.control_shape, - metadata.application, - ), - ) + .map(|(round, commitment, creation_time_ns, control_shape)| { + (round, (commitment, creation_time_ns, control_shape)) }) .collect::>(), Err(error) => { @@ -783,80 +773,29 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( return; } }; - let mut assigned_applications = BTreeSet::new(); let durable_round = core.local_carrier_round(); if mode.is_autonomous() { - for (carrier_round, (commitment, _, control_shape, application)) in &persisted_local { - if !*control_shape { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::AutonomousWalContainsInvalidCarrier( - *carrier_round, - ) - .to_string(), - }) - .await; - return; - } - let Some(application) = application else { - if *commitment != crate::crypto::TransactionsCommitment::default() { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::AutonomousWalContainsInvalidCarrier( - *carrier_round, - ) - .to_string(), - }) - .await; - return; - } - continue; - }; - let Some(recovered) = pending_local.get(&application.round) else { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::MissingRecoveredLocalHeader( - application.round, - ) - .to_string(), - }) - .await; - return; - }; - if recovered.application_header.reference() != *application - || recovered.transactions_commitment != *commitment - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch( - application.round, - ) - .to_string(), - }) - .await; - return; - } - assigned_applications.insert(*application); + if let Some((round, _)) = + persisted_local + .iter() + .find(|(_, (commitment, _, control_shape))| { + *commitment != crate::crypto::TransactionsCommitment::default() + || !*control_shape + }) + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::AutonomousWalContainsApplicationCarrier( + *round, + ) + .to_string(), + }) + .await; + return; } - pending_local.retain(|_, local| { - !assigned_applications.contains(&local.application_header.reference()) - }); } else { - for (round, (commitment, creation_time_ns, _, application)) in &persisted_local { - if application.is_some() { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) - .to_string(), - }) - .await; - return; - } + for (round, (commitment, creation_time_ns, _)) in &persisted_local { let Some(recovered) = pending_local.get(round) else { let _ = startup_events .send(ShadowServiceEventV1::Rejected { @@ -894,7 +833,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( .await; return; } - pending_local.retain(|round, _| *round >= core.local_carrier_round()); } let reported_shadow_deliveries = match core.delivered_identities() { Ok(identities) => identities.into_iter().collect::>(), @@ -909,26 +847,12 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }; let recovered_shadow_deliveries = reported_shadow_deliveries.clone(); - let reported_application_deliveries = match core.delivered_application_headers() { - Ok(headers) => headers - .into_iter() - .map(|(_, header)| header.reference()) - .collect(), - Err(error) => { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: error.to_string(), - }) - .await; - return; - } - }; let reported_shadow_delivery_slots = reported_shadow_deliveries .iter() .map(delivery_slot) .collect(); let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); + pending_local.retain(|round, _| *round >= core.local_carrier_round()); let sync_round = core.local_carrier_round(); let state = ShadowServiceStateV1 { core, @@ -943,7 +867,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( observed_topology: BTreeMap::new(), invalidated_by_overload: actor_invalidated_by_overload, pending_local, - assigned_applications, pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), sync_last_attempt: BTreeMap::new(), @@ -956,7 +879,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( heartbeat_notification_pending: actor_heartbeat_notification_pending, direct_deliveries: BTreeSet::new(), reported_shadow_deliveries, - reported_application_deliveries, recovered_shadow_deliveries, comparison_backlog, reported_matches: BTreeSet::new(), @@ -1071,7 +993,6 @@ struct ShadowServiceStateV1 { observed_topology: BTreeMap, invalidated_by_overload: Arc>>, pending_local: BTreeMap, - assigned_applications: BTreeSet, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, @@ -1084,7 +1005,6 @@ struct ShadowServiceStateV1 { heartbeat_notification_pending: Arc, direct_deliveries: BTreeSet, reported_shadow_deliveries: BTreeSet, - reported_application_deliveries: BTreeSet, recovered_shadow_deliveries: BTreeSet, comparison_backlog: ShadowComparisonBacklogV1, reported_matches: BTreeSet, @@ -1294,7 +1214,7 @@ impl ShadowServiceStateV1 { self.emit_clock_state(); } - fn try_create_autonomous_carrier(&mut self) { + fn try_create_autonomous_heartbeat(&mut self) { if !self.mode.is_autonomous() || !self.core.can_create_carrier() { self.emit_clock_state(); return; @@ -1306,27 +1226,10 @@ impl ShadowServiceStateV1 { .try_into() .unwrap_or(TimestampNs::MAX); let before = self.core.wal_counts(); - let application_round = self.pending_local.keys().next().copied(); - let application = application_round.and_then(|round| self.pending_local.remove(&round)); - let result = match &application { - Some(application) => self.core.create_local_application_carrier( - application.application_header.clone(), - creation_time_ns, - ), - None => self.core.create_local_control_heartbeat(creation_time_ns), - }; - match result { + match self.core.create_local_control_heartbeat(creation_time_ns) { Ok((envelope, effects)) => { - if let Some(application) = application { - self.assigned_applications - .insert(application.application_header.reference()); - } self.emit(ShadowServiceEventV1::Input { - kind: if application_round.is_some() { - "application_carrier" - } else { - "heartbeat" - }, + kind: "heartbeat", outcome: "accepted", }); self.report_wal_delta(before); @@ -1334,9 +1237,6 @@ impl ShadowServiceStateV1 { self.process_effects(effects); } Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { - if let Some(application) = application { - self.pending_local.insert(application.round, application); - } // The local slot is open syntactically but cannot yet name a // quorum of exact previous-round admitted parents. A later // authenticated ingress or timer tick retries it. @@ -1346,12 +1246,7 @@ impl ShadowServiceStateV1 { }); self.emit_clock_state(); } - Err(error) => { - if let Some(application) = application { - self.pending_local.insert(application.round, application); - } - self.mark_fatal(error); - } + Err(error) => self.mark_fatal(error), } } @@ -1363,7 +1258,7 @@ impl ShadowServiceStateV1 { fn drive_autonomous_catch_up(&mut self) { while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(); + self.try_create_autonomous_heartbeat(); if self.core.local_carrier_round() == round_before { break; } @@ -1391,37 +1286,6 @@ impl ShadowServiceStateV1 { ); return; } - if self.mode.is_autonomous() { - let application_reference = local.application_header.reference(); - if self.assigned_applications.contains(&application_reference) { - self.emit(ShadowServiceEventV1::Input { - kind: "application", - outcome: "already_assigned", - }); - return; - } - if let Some(existing) = self.pending_local.get(&local.round) { - if existing == &local { - self.emit(ShadowServiceEventV1::Input { - kind: "application", - outcome: "duplicate", - }); - } else { - self.reject( - None, - ShadowServiceErrorV1::ConflictingLocalHeader(local.round), - ); - } - return; - } - self.pending_local.insert(local.round, local); - self.emit(ShadowServiceEventV1::Input { - kind: "application", - outcome: "queued", - }); - self.retry_pending_local(); - return; - } let durable_round = self.core.local_carrier_round(); if local.round < durable_round { self.emit(ShadowServiceEventV1::Input { @@ -1460,19 +1324,6 @@ impl ShadowServiceStateV1 { /// advances the model; recovered historical headers below that clock are /// harmless idempotent replays. fn retry_pending_local(&mut self) { - if self.mode.is_autonomous() { - while (!self.pending_local.is_empty() || self.core.has_pending_application_phase_work()) - && self.core.can_create_carrier() - && !self.fatal - { - let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_carrier(); - if self.core.local_carrier_round() == round_before { - break; - } - } - return; - } loop { let durable_round = self.core.local_carrier_round(); self.pending_local @@ -1739,7 +1590,6 @@ impl ShadowServiceStateV1 { } self.report_wal_delta(before); self.process_effects(outcome.effects().to_vec()); - self.retry_pending_local(); if self .core .admitted_reference(response.author, response.round) @@ -1793,21 +1643,6 @@ impl ShadowServiceStateV1 { self.emit_slot_comparison(slot); self.emit_comparison_backlog(); } - let applications = match self.core.delivered_application_headers() { - Ok(applications) => applications, - Err(error) => { - self.reject(None, error); - return; - } - }; - for (carrier, header) in applications { - if self - .reported_application_deliveries - .insert(header.reference()) - { - self.emit(ShadowServiceEventV1::EmbeddedApplicationDelivered { carrier, header }); - } - } } fn emit_slot_comparison(&mut self, slot: ShadowDeliverySlotV1) { @@ -1881,9 +1716,10 @@ fn run_shadow_service( }); state.emit_comparison_backlog(); state.process_effects(open_report.recovery_effects().to_vec()); - state.retry_pending_local(); if state.mode.is_autonomous() { state.emit_clock_state(); + } else { + state.retry_pending_local(); } } @@ -2078,7 +1914,7 @@ fn run_shadow_service( state.flush_recovery_requests(); state.flush_carrier_sync_requests(false); } - ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_carrier(), + ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_heartbeat(), ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); @@ -2277,7 +2113,6 @@ mod tests { authority, self.context, ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), - Vec::new(), heartbeat_interval, wal_sync_policy, ) @@ -2367,7 +2202,6 @@ mod tests { events: &mut [mpsc::Receiver], open_rounds: &mut [RoundNumber], deliveries: &mut [usize], - application_deliveries: &mut [BTreeSet], sync_requests: &mut usize, target_open_round: RoundNumber, ) { @@ -2421,12 +2255,6 @@ mod tests { ShadowServiceEventV1::Delivered(_) => { deliveries[sender] = deliveries[sender].saturating_add(1); } - ShadowServiceEventV1::EmbeddedApplicationDelivered { - header, - .. - } => { - application_deliveries[sender].insert(header.reference()); - } ShadowServiceEventV1::Rejected { error, .. } if error.contains("FutureCarrierOutsideBuffer") || error.contains("unexpected shadow response") => {} @@ -2595,7 +2423,6 @@ mod tests { own_prev: carrier_genesis_reference(author), weak_parents, transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - application_header: None, data_acknowledgments: Vec::new(), phase_batch: Vec::new(), consensus_vertex: None, @@ -2684,7 +2511,6 @@ mod tests { let mut open_rounds = vec![1; n]; let mut deliveries = vec![0; n]; - let mut application_deliveries = vec![BTreeSet::new(); n]; let mut sync_requests = 0; for (authority, handle) in handles.iter().enumerate() { for peer in 0..n { @@ -2702,7 +2528,6 @@ mod tests { &mut events, &mut open_rounds, &mut deliveries, - &mut application_deliveries, &mut sync_requests, fixed_round + 1, ) @@ -2728,100 +2553,6 @@ mod tests { } } - #[tokio::test] - async fn autonomous_application_header_wins_the_open_carrier_slot_and_uses_v2() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start_autonomous(0); - wait_ready(&mut events).await; - let application = direct_header(0, 1, 0x6A); - handle.local_header(&application).unwrap(); - - let envelope = next_carrier(&mut events, 1).await; - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &envelope.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - assert_eq!(candidate.header().carrier_round(), 1); - assert_eq!(candidate.header().application_header(), Some(&application)); - assert_eq!( - envelope.canonical_carrier[1], - crate::starfish_rbc_dag::CARRIER_WIRE_FORMAT_VERSION_V2 - ); - - handle.shutdown().await.unwrap(); - task.await.unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn embedded_rbc_delivers_every_application_header_from_the_carrier_dag() { - let harness = Harness::new(); - let mut handles = Vec::new(); - let mut events = Vec::new(); - let mut tasks = Vec::new(); - for authority in 0..N as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous(authority); - wait_ready(&mut node_events).await; - handles.push(handle); - events.push(node_events); - tasks.push(task); - } - for (authority, handle) in handles.iter().enumerate() { - for peer in 0..N { - if peer != authority { - handle.peer_connected(peer as AuthorityIndex).unwrap(); - } - } - } - - let applications = (0..N as AuthorityIndex) - .map(|authority| direct_header(authority, 1, 0x70 + authority as u8)) - .collect::>(); - let expected = applications - .iter() - .map(RbcCanonicalHeader::reference) - .collect::>(); - for (handle, application) in handles.iter().zip(&applications) { - handle.local_header(application).unwrap(); - } - - let mut open_rounds = vec![1; N]; - let mut deliveries = vec![0; N]; - let mut application_deliveries = vec![BTreeSet::new(); N]; - let mut sync_requests = 0; - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut application_deliveries, - &mut sync_requests, - 5, - ) - .await; - - assert!( - application_deliveries - .iter() - .all(|delivered| delivered == &expected), - "every node must deliver every exact embedded application: {application_deliveries:?}" - ); - assert!( - open_rounds.iter().all(|round| *round >= 5), - "application-critical phase carriers must not wait for a heartbeat tick" - ); - assert_eq!(sync_requests, 0); - - drop(events); - for handle in &handles { - handle.shutdown().await.unwrap(); - } - for task in tasks { - task.await.unwrap(); - } - } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn four_node_autonomous_zero_load_clock_delivers_mature_heartbeats() { assert_autonomous_zero_load_progress(4).await; @@ -2963,7 +2694,6 @@ mod tests { } let mut open_rounds = vec![1; N]; let mut deliveries = vec![0; N]; - let mut application_deliveries = vec![BTreeSet::new(); N]; let mut sync_requests = 0; for handle in &handles { handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); @@ -2973,7 +2703,6 @@ mod tests { &mut events, &mut open_rounds, &mut deliveries, - &mut application_deliveries, &mut sync_requests, 2, ) @@ -3024,7 +2753,6 @@ mod tests { &mut events, &mut open_rounds, &mut deliveries, - &mut application_deliveries, &mut sync_requests, 10, ) @@ -3101,86 +2829,6 @@ mod tests { stop(restarted, restarted_events, restarted_task).await; } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn autonomous_wal_restart_reconciles_and_replays_exact_application_origin() { - let harness = Harness::new(); - let application = direct_header(0, 1, 0x7B); - let (handle, mut events, task) = harness.start_autonomous(0); - wait_ready(&mut events).await; - handle.local_header(&application).unwrap(); - let original = next_carrier(&mut events, 1).await; - stop(handle, events, task).await; - - let (restarted, mut restarted_events, restarted_task) = - start_starfish_rbc_dag_autonomous_clock_service_v1( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - vec![application.clone()], - Duration::from_secs(60 * 60), - ShadowWalSyncPolicyV1::EveryBatch, - ) - .unwrap(); - loop { - match next_event(&mut restarted_events).await { - ShadowServiceEventV1::Ready { autonomous_clock } => { - assert!(autonomous_clock); - break; - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("application WAL restart failed: {error}") - } - _ => {} - } - } - restarted.local_header(&application).unwrap(); - restarted - .carrier_sync_request( - 1, - RbcDagShadowCarrierSyncRequest { - author: 0, - round: 1, - }, - ) - .unwrap(); - loop { - if let ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrierSyncResponse(response), - } = next_event(&mut restarted_events).await - { - assert_eq!(response.canonical_carrier, original.canonical_carrier); - assert_eq!( - response.authentication_sidecar, - original.authentication_sidecar - ); - break; - } - } - stop(restarted, restarted_events, restarted_task).await; - - let (_invalid, invalid_events, invalid_task) = - start_starfish_rbc_dag_autonomous_clock_service_v1( - &harness.paths[0], - harness.committee.clone(), - 0, - harness.context, - ShadowAuthorizerV1::MacVector(harness.keyrings[0].clone()), - Vec::new(), - Duration::from_secs(60 * 60), - ShadowWalSyncPolicyV1::EveryBatch, - ) - .unwrap(); - assert!( - startup_rejection(invalid_events) - .await - .contains("no matching recovered direct header") - ); - invalid_task.await.unwrap(); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn buffered_wal_reports_append_without_durability_and_reopens_after_clean_shutdown() { let harness = Harness::new(); @@ -3251,7 +2899,6 @@ mod tests { transactions_commitment: TransactionsCommitment::from_bytes( [0xB0 + author as u8; 32], ), - application_header: None, data_acknowledgments: Vec::new(), phase_batch: vec![statement], consensus_vertex: None, diff --git a/crates/starfish-core/src/starfish_rbc_service.rs b/crates/starfish-core/src/starfish_rbc_service.rs index 5165b3b8..2b536e78 100644 --- a/crates/starfish-core/src/starfish_rbc_service.rs +++ b/crates/starfish-core/src/starfish_rbc_service.rs @@ -207,10 +207,8 @@ pub(crate) struct RbcServiceHandle { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum RbcPhaseAuthorityV1 { Direct, - EmbeddedCarrierDag, EmbeddedSingleDag { echo_qc_fast_path: bool }, } - impl RbcServiceHandle { #[allow(dead_code)] pub(crate) async fn start_local_header( @@ -359,6 +357,7 @@ pub(crate) fn start_starfish_rbc_service( ) } +#[allow(clippy::too_many_arguments)] pub(crate) fn start_starfish_rbc_service_with_phase_authority( committee: Arc, own_authority: AuthorityIndex, @@ -390,7 +389,7 @@ pub(crate) fn start_starfish_rbc_service_with_phase_authority( let echo_qc_fast_path = match phase_authority { RbcPhaseAuthorityV1::EmbeddedSingleDag { echo_qc_fast_path } => echo_qc_fast_path, - RbcPhaseAuthorityV1::Direct | RbcPhaseAuthorityV1::EmbeddedCarrierDag => false, + RbcPhaseAuthorityV1::Direct => false, }; let kernel = StarfishRbcKernel::new_with_echo_qc_fast_path( committee.clone(), @@ -506,11 +505,9 @@ impl RbcServiceState { self.accept_direct_initial(peer, proposal); } RbcServiceMessage::Phase { peer, message } => { - if self.phase_authority == RbcPhaseAuthorityV1::Direct { - match self.kernel.handle_phase(peer, message) { - Ok(effects) => self.process_effects(effects), - Err(error) => self.reject(Some(peer), error.into()), - } + match self.kernel.handle_phase(peer, message) { + Ok(effects) => self.process_effects(effects), + Err(error) => self.reject(Some(peer), error.into()), } } RbcServiceMessage::HeaderRequest { peer, block_ref } => { @@ -786,9 +783,6 @@ impl RbcServiceState { for effect in effects { match effect { RbcEffect::MulticastPhase { phase, block_ref } => { - if self.phase_authority == RbcPhaseAuthorityV1::EmbeddedCarrierDag { - continue; - } if matches!( self.phase_authority, RbcPhaseAuthorityV1::EmbeddedSingleDag { .. } @@ -821,9 +815,6 @@ impl RbcServiceState { self.note_pending_fetch(block_ref, holders); } RbcEffect::Deliver(header) => { - if self.phase_authority == RbcPhaseAuthorityV1::EmbeddedCarrierDag { - continue; - } self.pending_fetches.remove(&header.reference()); let _ = self.events.send(RbcServiceEvent::Delivered(header)); } @@ -1046,27 +1037,6 @@ mod tests { .unwrap() } - fn start_embedded_phase_service() -> ( - RbcServiceHandle, - mpsc::UnboundedReceiver, - JoinHandle<()>, - ) { - let committee = Committee::new_test(vec![1; 4]); - let keyrings = mac_keyrings_for_test(4); - start_starfish_rbc_service_with_phase_authority( - committee, - 0, - instance(), - BlockAuthenticationScheme::MacVector, - Arc::new(keyrings[0].clone()), - RbcInitialAuthenticator::Mac, - 1, - Duration::from_secs(3_600), - RbcPhaseAuthorityV1::EmbeddedCarrierDag, - ) - .unwrap() - } - fn start_single_dag_service() -> ( RbcServiceHandle, mpsc::UnboundedReceiver, @@ -1089,7 +1059,6 @@ mod tests { ) .unwrap() } - async fn next_event(events: &mut mpsc::UnboundedReceiver) -> RbcServiceEvent { tokio::time::timeout(Duration::from_secs(2), events.recv()) .await @@ -1164,37 +1133,6 @@ mod tests { } } - #[tokio::test] - async fn embedded_phase_authority_emits_init_but_no_direct_echo_or_delivery() { - let (handle, mut events, task) = start_embedded_phase_service(); - let canonical = handle.start_local_header(local_header(1, 4)).await.unwrap(); - let mut staged = false; - let mut initials = 0; - for _ in 0..4 { - match next_event(&mut events).await { - RbcServiceEvent::HeaderStaged(header) => { - assert_eq!(header.reference(), canonical.reference()); - staged = true; - } - RbcServiceEvent::Network { - message: NetworkMessage::RbcInitial(_), - .. - } => initials += 1, - event => panic!("unexpected init-only event: {event:?}"), - } - } - assert!(staged); - assert_eq!(initials, 3); - assert!( - tokio::time::timeout(Duration::from_millis(25), events.recv()) - .await - .is_err(), - "direct ECHO/READY or delivery escaped the embedded authority boundary" - ); - drop(handle); - task.await.unwrap(); - } - #[tokio::test] async fn single_dag_phase_authority_emits_typed_reference_not_phase_message() { let (handle, mut events, task) = start_single_dag_service(); @@ -1232,7 +1170,6 @@ mod tests { drop(handle); task.await.unwrap(); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blocking_local_start_waits_for_kernel_selection_and_event_enqueue() { let (handle, mut events, task) = start_service(0, BlockAuthenticationScheme::Ed25519); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 0acf9f1d..10695e4a 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -59,16 +59,6 @@ impl Validator { "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" )); } - if public_config - .parameters - .starfish_rbc_dag_embedded_rbc_authority - && (!public_config.parameters.starfish_rbc_dag_shadow - || !public_config.parameters.starfish_rbc_dag_autonomous_clock) - { - return Err(eyre!( - "Starfish-RBC-DAG embedded RBC authority requires the autonomous RBC-DAG shadow" - )); - } if public_config .parameters .starfish_rbc_single_dag_echo_qc_fast_path @@ -512,6 +502,41 @@ mod smoke_tests { })); } + #[tokio::test] + async fn autonomous_clock_rejects_zero_heartbeat_interval() { + let committee_size = 4; + let committee = Committee::new_for_benchmarks(committee_size); + let mut public_config = NodePublicConfig::new_for_tests(committee_size); + public_config.parameters.starfish_rbc_dag_shadow = true; + public_config.parameters.starfish_rbc_dag_autonomous_clock = true; + public_config + .parameters + .starfish_rbc_dag_heartbeat_interval_ms = 0; + public_config + .parameters + .refresh_starfish_rbc_protocol_instance(); + let private_config = + NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) + .remove(0); + + let result = Validator::start( + 0, + committee, + public_config, + private_config, + Parameters::default(), + "honest".to_string(), + "starfish-rbc".to_string(), + ) + .await; + + assert!(result.is_err_and(|error| { + error + .to_string() + .contains("autonomous heartbeat interval must be greater than zero") + })); + } + async fn run_commit_test( consensus: &str, block_authentication: Option<&str>, @@ -532,7 +557,6 @@ mod smoke_tests { port_offset, starfish_rbc_dag_shadow, false, - false, ) .await; } @@ -543,7 +567,6 @@ mod smoke_tests { port_offset: u16, starfish_rbc_dag_shadow: bool, autonomous_clock: bool, - embedded_rbc_authority: bool, ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); @@ -552,20 +575,17 @@ mod smoke_tests { public_config.parameters.block_authentication = block_authentication.map(str::to_string); public_config.parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; public_config.parameters.starfish_rbc_dag_autonomous_clock = autonomous_clock; - public_config - .parameters - .starfish_rbc_dag_embedded_rbc_authority = embedded_rbc_authority; + if autonomous_clock { + public_config + .parameters + .starfish_rbc_dag_heartbeat_interval_ms = 50; + } if consensus == "starfish-rbc" { public_config .parameters .refresh_starfish_rbc_protocol_instance(); } - let mut parameters = Parameters::default(); - if autonomous_clock { - // Exercise the shared Starfish/carrier pacemaker contract without - // making the integration test wait for production timeouts. - parameters.leader_timeout = Some(Duration::from_millis(50)); - } + let parameters = Parameters::default(); let dir = TempDir::new().unwrap(); let private_configs = NodePrivateConfig::new_for_benchmarks(dir.as_ref(), committee_size); @@ -628,12 +648,6 @@ mod smoke_tests { .with_label_values(&["delivery", "shadow"]) .get() > 0 - && (!embedded_rbc_authority - || metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "embedded_application"]) - .get() - > 0) && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 }) { break; @@ -653,32 +667,6 @@ mod smoke_tests { 0, "autonomous mode must not claim direct-round comparison" ); - if embedded_rbc_authority { - assert!( - metrics - .network_message_bytes_sent_total - .with_label_values(&["rbc_initial"]) - .get() - > 0, - "direct INIT remains the application/payload transport" - ); - assert_eq!( - metrics - .network_message_bytes_sent_total - .with_label_values(&["rbc_echo"]) - .get(), - 0, - "direct RBC ECHO must be disabled under embedded authority" - ); - assert_eq!( - metrics - .network_message_bytes_sent_total - .with_label_values(&["rbc_ready"]) - .get(), - 0, - "direct RBC READY must be disabled under embedded authority" - ); - } } } else if starfish_rbc_dag_shadow { let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR @@ -824,13 +812,7 @@ mod smoke_tests { #[tokio::test] async fn starfish_rbc_dag_autonomous_clock_advances_without_owning_consensus() { - run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1700, true, true, false) - .await; - } - - #[tokio::test] - async fn starfish_rbc_dag_embedded_rbc_is_the_only_phase_authority() { - run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1740, true, true, true).await; + run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1700, true, true).await; } #[tokio::test] diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 236357c4..236cdb06 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -80,10 +80,9 @@ enum Operation { /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, - /// Let embedded carrier ECHO/READY delivery certify application - /// headers. Requires the autonomous RBC-DAG mode. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_embedded_rbc_authority: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_heartbeat_interval_ms: Option, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -125,10 +124,9 @@ enum Operation { /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, - /// Let embedded carrier ECHO/READY delivery certify application - /// headers. Requires the autonomous RBC-DAG mode. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_embedded_rbc_authority: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_heartbeat_interval_ms: Option, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -192,10 +190,6 @@ enum Operation { /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. #[clap(long, default_value_t = false)] starfish_rbc_dag_autonomous_clock: bool, - /// Let embedded carrier ECHO/READY delivery certify application - /// headers. Requires the autonomous RBC-DAG mode. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_embedded_rbc_authority: bool, /// Testbed-only: deliver a single-DAG RBC header after a receiver-local /// quorum ECHO. This preserves uniqueness but not Byzantine /// selective-withholding totality, so it is restricted to finite @@ -242,7 +236,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_heartbeat_interval_ms, } => { run( authority, @@ -255,7 +249,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_heartbeat_interval_ms, ) .await? } @@ -272,7 +266,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_heartbeat_interval_ms, data_dir, base_ip, storage_backend, @@ -294,7 +288,7 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_embedded_rbc_authority, + starfish_rbc_dag_heartbeat_interval_ms, data_dir, base_ip, storage_backend, @@ -318,7 +312,6 @@ async fn main() -> Result<()> { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_shadow_buffered_wal, duration_secs, @@ -333,8 +326,6 @@ async fn main() -> Result<()> { node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; - node_parameters.starfish_rbc_dag_embedded_rbc_authority = - starfish_rbc_dag_embedded_rbc_authority; node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = starfish_rbc_single_dag_echo_qc_fast_path; node_parameters.starfish_rbc_dag_shadow_buffered_wal = @@ -462,12 +453,6 @@ async fn local_benchmark( } ); } - if node_parameters.starfish_rbc_dag_autonomous_clock { - println!( - "Carrier idle timeout: {} ms (shared Starfish leader pacemaker)", - node_parameters.leader_timeout.as_millis() - ); - } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( "Single-DAG receiver-local quorum-ECHO: ENABLED (signature-free latency lower bound; Byzantine totality not provided)" @@ -511,8 +496,6 @@ async fn local_benchmark( let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; let starfish_rbc_dag_autonomous_clock_expected = node_parameters.starfish_rbc_dag_autonomous_clock; - let starfish_rbc_dag_embedded_rbc_authority_expected = - node_parameters.starfish_rbc_dag_embedded_rbc_authority; // Create temporary directories for each validator let base_dir = PathBuf::from("local-benchmark"); @@ -685,7 +668,6 @@ async fn local_benchmark( committee_size, starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, - starfish_rbc_dag_embedded_rbc_authority_expected, autonomous_clock_baselines.clone(), Some(counter_baselines.clone()), ); @@ -714,7 +696,6 @@ async fn local_benchmark( committee_size, starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, - starfish_rbc_dag_embedded_rbc_authority_expected, autonomous_clock_baselines, Some(counter_baselines), ); @@ -736,7 +717,7 @@ async fn run( block_authentication: Option, starfish_rbc_dag_shadow: bool, starfish_rbc_dag_autonomous_clock: bool, - starfish_rbc_dag_embedded_rbc_authority: bool, + starfish_rbc_dag_heartbeat_interval_ms: Option, ) -> Result<()> { tracing::info!("Starting node {authority}"); @@ -754,10 +735,10 @@ async fn run( if starfish_rbc_dag_autonomous_clock { public_config.parameters.starfish_rbc_dag_autonomous_clock = true; } - if starfish_rbc_dag_embedded_rbc_authority { + if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { public_config .parameters - .starfish_rbc_dag_embedded_rbc_authority = true; + .starfish_rbc_dag_heartbeat_interval_ms = interval_ms; } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" @@ -798,7 +779,7 @@ async fn dryrun( block_authentication: Option, starfish_rbc_dag_shadow: bool, starfish_rbc_dag_autonomous_clock: bool, - starfish_rbc_dag_embedded_rbc_authority: bool, + starfish_rbc_dag_heartbeat_interval_ms: Option, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -843,8 +824,9 @@ async fn dryrun( node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; - node_parameters.starfish_rbc_dag_embedded_rbc_authority = - starfish_rbc_dag_embedded_rbc_authority; + if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + } ensure_starfish_rbc_protocol_instance(&consensus_protocol, &mut node_parameters); if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; @@ -1084,7 +1066,6 @@ mod tests { "mac", "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", - "--starfish-rbc-dag-embedded-rbc-authority", "--starfish-rbc-single-dag-echo-qc-fast-path", "--starfish-rbc-dag-shadow-buffered-wal", ]) @@ -1095,7 +1076,6 @@ mod tests { block_authentication, starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_embedded_rbc_authority, starfish_rbc_single_dag_echo_qc_fast_path, starfish_rbc_dag_shadow_buffered_wal, .. @@ -1107,7 +1087,6 @@ mod tests { assert_eq!(block_authentication.as_deref(), Some("mac")); assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); - assert!(starfish_rbc_dag_embedded_rbc_authority); assert!(starfish_rbc_single_dag_echo_qc_fast_path); assert!(starfish_rbc_dag_shadow_buffered_wal); } @@ -1117,6 +1096,7 @@ mod tests { let mut parameters = NodeParameters { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, + starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }; @@ -1129,5 +1109,6 @@ mod tests { ); assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); + assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 3a01c8fb..15280547 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -11,15 +11,14 @@ committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoi full validator recovery remain incomplete The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not -implemented. The staged prototype runs under `starfish-rbc`: direct-header comparison uses -`--starfish-rbc-dag-shadow`, the independent carrier clock adds -`--starfish-rbc-dag-autonomous-clock`, and milestone five makes embedded carrier ECHO/READY the -only application-header certification authority with -`--starfish-rbc-dag-embedded-rbc-authority`. Direct INIT still transports the application payload, -but direct ECHO, READY, and delivery are suppressed in that mode. Performance experiments may add -`--starfish-rbc-dag-shadow-buffered-wal`; that profile is explicitly not crash-safe. Consensus -projection, commit, and output still use the existing Starfish DAG. The eventual protocol is new, -not a transport option or a version-two alias for `starfish-rbc`. +implemented. The milestone-three direct-header comparison runtime is enabled with `--consensus +starfish-rbc --starfish-rbc-dag-shadow`. Milestone four adds a separate control-only runtime with +`--starfish-rbc-dag-autonomous-clock`; its carrier rounds advance independently through +authenticated admission and empty heartbeats. Performance experiments may add +`--starfish-rbc-dag-shadow-buffered-wal` to remove per-transition disk synchronization; that +profile is explicitly not crash-safe. Both modes leave the direct prototype's DAG, +pacemaker, commit, and output unchanged. The eventual protocol is new, not a transport option or a +version-two alias for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -54,24 +53,20 @@ carrier/RBC, certified-projection, decision, and crash-journal models. Milestone opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired direct/shadow delivery observations. Milestone four adds an independently authenticated autonomous heartbeat namespace, sequential `Q`-admitted carrier clock, bounded future buffering, exact-slot -synchronization, and clock-state metrics. Milestone five adds a version-two application carrier -containing the exact canonical application header, durable application-origin reconciliation, -immediate application/phase scheduling, and an opt-in authority boundary that prevents direct -ECHO/READY from certifying a header. Idle control heartbeats use the same resolved leader timeout as -the Starfish pacemaker (600 ms for Push/Starfish-RBC by default); application carriers and their -encodable ECHO/READY follow-ups are event-driven and do not wait for that timeout. - -The current authoritative mode changes only header certification. Consensus vertices, certified -projection, commits, and application ordering remain on the existing Starfish DAG. Direct INIT is -still the application-payload transport and is not a certification vote. Replacing that remaining -wrapper with a payload-only path and moving consensus into the clean carrier projection are later -milestones. +synchronization, and clock-state metrics. The direct `starfish-rbc` service remains the only +authority: shadow admission, delivery, recovery, clock advancement, or failure cannot advance a +proposal, mark a DAG vertex clean, vote, commit, or order output. + +Autonomous mode is intentionally control-only at this milestone. It ignores direct application +headers and uses a distinct WAL and authentication protocol instance. This avoids falsely equating +direct consensus rounds with faster carrier rounds, but also means milestone four does not yet +measure application latency through the new carrier DAG. Application-origin assignment and the +certified consensus projection remain later milestones. Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode -requires an identical recovered direct-header history, control-only autonomous history reopens -without direct headers, and autonomous application origins must match recovered direct history. -Every mode serves byte-identical exact-slot responses. This is not a full validator crash-recovery -claim. The authoritative direct +requires an identical recovered direct-header history, while autonomous mode reopens its +control-only heartbeat history without direct headers and serves byte-identical exact-slot +responses. This is not a full validator crash-recovery claim. The authoritative direct `starfish-rbc` baseline does not yet durably record its remote-slot ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest @@ -920,18 +915,15 @@ cannot be used for crash-safety claims. Benchmark output reports appended and du separately. The clone-based reducer remains intentionally unoptimized until measurement shows it matters. -A matched 10-validator local sequence on 2026-08-11 used a full 60-second active transaction -window, the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and the buffered -benchmark WAL. Milestone-five idle carriers use the same resolved 600 ms Push leader timeout as -Starfish-RBC; application and encodable phase carriers are immediate. The harness waits through -generator warmup, snapshots cumulative counters at the active boundary, and drains final latency -samples. +A matched 10-validator local A/B on 2026-08-11 used a full 60-second active transaction window, +the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous +heartbeat. The harness waits through generator warmup, snapshots cumulative counters at the active +boundary, and drains final latency samples. | Profile | Verdict | TPS | p50 block | p50 E2E | Outbound | |---|---:|---:|---:|---:|---:| | Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | | Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | -| Embedded RBC authoritative (milestone five) | VALID 10/10 | 861.92 | 3,539.3 ms | 5,477.5 ms | 0.52 MB/s | | Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted @@ -939,16 +931,8 @@ heartbeats, 27,180 embedded-RBC deliveries, 27,424 appended batches, zero round pending recovery. Its latency and throughput match the shadow-off baseline while making the expected extra carrier traffic visible. The crash-safe run shed shadow work and is reported only as a diagnostic: it isolates synchronous persistence as a severe observer effect and must not be -cited as a protocol result. - -The milestone-five run delivered 10,722 application headers through embedded RBC, reached carrier -rounds 458–459, and ended with zero pending recovery. Its direct ECHO and READY paths were disabled; -the composed four-validator test also asserts zero such outbound messages. Its higher latency is -not evidence for a bad 250 ms timer—the independent timer was removed, and the same Starfish -timeout was used. It exposes the transitional clean-predecessor gate: the existing direct Starfish -DAG still serializes proposal creation on embedded delivery. Milestone six must replace that gate -with the optimistic carrier clock plus certified consensus projection before the complete protocol -can be expected to recover Starfish pipelining. +cited as a protocol result. Neither run measures application latency through the carrier DAG yet, +because the direct Starfish-RBC path remains authoritative in milestone four. ## 19. Contained implementation milestones @@ -970,11 +954,10 @@ Every milestone is committed separately. 4. **Optimistic carrier clock (implemented, opt-in control shadow):** run a separately namespaced, control-only heartbeat carrier plane with the distinct authenticated-admission latch, sequential quorum clock, bounded future buffer, exact-slot synchronization, durable restart, and clock - validity metrics while consensus still uses the current direct baseline. -5. **Authoritative embedded RBC (implemented, opt-in):** encode exact canonical application headers - in version-two carriers, durably reconcile their origins, schedule application/phase carriers - immediately, and remove direct ECHO/READY/delivery authority. Direct INIT remains payload - transport; composed tests assert zero direct ECHO/READY traffic and positive embedded delivery. + validity metrics while consensus still uses the current direct baseline. Application headers are + not assigned to autonomous carrier rounds yet. +5. **Authoritative embedded RBC:** remove direct ECHO/READY authority only after shadow tests show + identical delivery under reordering, loss, equivocation, poisoned tags, and restart. 6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit leader choice, contiguous delivery frontiers, and strict clean-only committer consumers. 7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct @@ -991,8 +974,8 @@ the executable model or measured prototype: - production maximum future-carrier buffer and payload runahead (the executable model deliberately uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); -- whether the shared Starfish leader-timeout policy needs a separately proved adaptive low-load - rule; the prototype intentionally does not introduce a second heartbeat timeout; +- the production control-heartbeat rate under low load and backpressure (the autonomous shadow's + configurable 250 ms default is an empirical test value, not a protocol constant); - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and From 4f3af513ac4b5fd5e3458e32c18ddefdf1743edb Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:44:19 +0200 Subject: [PATCH 57/62] Revert "Isolate RBC-DAG benchmark persistence overhead" This reverts commit d6f11cd29318c98fd4319ac1dff460bbd78abde2. --- README.md | 52 +--- crates/orchestrator/src/benchmark.rs | 7 +- crates/orchestrator/src/main.rs | 12 - crates/orchestrator/src/measurements.rs | 75 +----- crates/orchestrator/src/protocol/starfish.rs | 8 - crates/starfish-core/src/config.rs | 43 ++-- crates/starfish-core/src/metrics.rs | 238 ++++-------------- crates/starfish-core/src/net_sync.rs | 35 +-- .../src/starfish_rbc_dag/storage.rs | 75 +----- .../src/starfish_rbc_dag_shadow.rs | 23 +- .../src/starfish_rbc_dag_shadow_service.rs | 90 +------ crates/starfish-core/src/validator.rs | 61 +---- crates/starfish/src/main.rs | 73 ++---- docs/starfish-rbc-dag-protocol.md | 69 ++--- 14 files changed, 146 insertions(+), 715 deletions(-) diff --git a/README.md b/README.md index a0fdb39f..8d6c8ded 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,8 @@ authoritative in both modes, and shadow failures or results cannot affect propos output as protocol state. Shadow traffic still shares the validator's network socket and bandwidth, so it can perturb timing, and it must be enabled only on a homogeneous new-binary committee; there is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` selector is not -implemented yet. The default shadow uses per-transition fsync and a clone-based reference reducer, -so it is a correctness instrument, not a fair performance baseline, and carries no safety or -liveness claim. Benchmark runs may add `--starfish-rbc-dag-shadow-buffered-wal`; this preserves the -ordered checksummed log and syncs it on clean shutdown, but deliberately gives up crash safety for -that run. Appended and durably synchronized WAL records are reported separately. +implemented yet. The shadow uses per-transition fsync and a clone-based reference reducer, so it is +a correctness instrument, not a fair performance baseline, and carries no safety or liveness claim. Its WAL can reopen the shadow actor, but this is not full validator crash recovery: authoritative direct Starfish-RBC phase and delivery locks are not durable yet, so that baseline remains fail-stop across process restart. The design and proof obligations are documented in the @@ -75,31 +72,19 @@ observational path was disabled or shed work and the comparison must be discarde production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between the cumulative direct and shadow delivery counters. Autonomous runs instead require -`starfish_rbc_dag_shadow_clock_valid == 1`, heartbeat/WAL progress, advancing carrier rounds, +`starfish_rbc_dag_shadow_clock_valid == 1`, durable heartbeat progress, advancing carrier rounds, in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports at most 60 validators in mirror mode and 20 in autonomous mode. -A matched 10-validator, 60-second-active-window local A/B on 2026-08-11 used the AWS RTT emulator, -nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. - -| Profile | Verdict | TPS | Block latency | E2E latency | Outbound BW | -|---|---:|---:|---:|---:|---:| -| Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | -| Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | - -Every shadow validator reached carrier round 275 with zero skew and pending recovery; the run -recorded 2,749 heartbeats, 27,180 embedded-RBC deliveries, and 27,424 WAL batches. Thus this -prototype's carrier/RBC work had no measurable latency penalty in this run; the extra shadow -traffic cost about 0.05 MB/s outbound. The result is not yet application latency through RBC-DAG -because direct Starfish-RBC remains authoritative. - -The same 250 ms experiment with the default crash-safe, per-transition-fsync WAL shed shadow work -and ended `INVALID` (9/10 valid validators); its authoritative path slowed to 2,569.1 ms block and -3,067.4 ms end-to-end latency. This isolates synchronous shadow persistence as the prior observer -effect, rather than a carrier-DAG latency regression. Use the buffered profile for protocol -benchmarks and the default profile for crash/replay tests; do not cite an `INVALID` run as a -protocol result. The local harness now starts its timer after transaction-generator warmup, -subtracts warmup counters, and drains the final latency samples. +An exploratory 10-validator, 60-second local run with the AWS RTT emulator, nominal 1,000 tx/s +load, MAC authentication, and a 250 ms autonomous heartbeat completed with a `VALID` clock +verdict on 2026-08-11. All validators reached carrier round 196 with zero skew and zero pending +recovery; the run recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative +direct Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 +end-to-end latency, and 0.45 MB/s average outbound bandwidth. This is a prototype continuity +result, not a fair performance comparison: the 60-second cutoff includes the local generator +warmup, and the control-only shadow still performs per-transition fsync/reference-model work while +sharing the authoritative socket. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's @@ -307,19 +292,6 @@ cargo run --release --bin starfish -- local-benchmark \ Additional flags: `--dissemination-mode`, `--adversarial-latency`, `--uniform-latency-ms`. -For the non-authoritative autonomous RBC-DAG benchmark profile: - -```bash -cargo run --release --bin starfish -- local-benchmark \ - --committee-size 10 --load 1000 --consensus starfish-rbc \ - --block-authentication mac --mimic-extra-latency \ - --starfish-rbc-dag-shadow --starfish-rbc-dag-autonomous-clock \ - --starfish-rbc-dag-heartbeat-interval-ms 250 \ - --starfish-rbc-dag-shadow-buffered-wal --duration-secs 60 -``` - -The buffered WAL is benchmark-only and is not crash-safe. - ### Local dryrun with monitoring and dashboard The dryrun script launches a Docker-based local testbed with diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index 7eae00ad..f29acaf9 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -116,8 +116,6 @@ pub struct BenchmarkRunSummary { #[serde(default)] pub shadow_delivery_ambiguous: usize, #[serde(default)] - pub shadow_wal_appended_records: usize, - #[serde(default)] pub shadow_wal_durable_records: usize, #[serde(default)] pub shadow_pending_recovery: usize, @@ -175,7 +173,7 @@ impl BenchmarkRunSummary { shadow_comparison_valid_nodes,shadow_direct_deliveries,shadow_deliveries,\ shadow_delivery_matches,\ shadow_delivery_mismatches,shadow_delivery_ambiguous,\ - shadow_wal_appended_records,shadow_wal_durable_records,shadow_pending_recovery,\ + shadow_wal_durable_records,shadow_pending_recovery,\ shadow_unpaired_direct,shadow_unpaired_shadow,\ shadow_unpaired_max_round_lag,\ shadow_autonomous_clock_enabled,shadow_autonomous_clock_valid,\ @@ -227,7 +225,6 @@ impl BenchmarkRunSummary { self.shadow_delivery_matches.to_string(), self.shadow_delivery_mismatches.to_string(), self.shadow_delivery_ambiguous.to_string(), - self.shadow_wal_appended_records.to_string(), self.shadow_wal_durable_records.to_string(), self.shadow_pending_recovery.to_string(), self.shadow_unpaired_direct.to_string(), @@ -856,7 +853,6 @@ pub mod test { #[test] fn benchmark_csv_includes_autonomous_clock_verdict_and_state() { let summary = BenchmarkRunSummary { - shadow_wal_appended_records: 9, shadow_autonomous_clock_enabled: true, shadow_autonomous_clock_valid: true, shadow_autonomous_clock_valid_nodes: 4, @@ -877,7 +873,6 @@ pub mod test { assert_eq!(headers.len(), values.len()); for (header, expected) in [ - ("shadow_wal_appended_records", "9"), ("shadow_autonomous_clock_enabled", "true"), ("shadow_autonomous_clock_valid", "true"), ("shadow_autonomous_clock_valid_nodes", "4"), diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index b00903d9..1c30ea52 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -77,11 +77,6 @@ pub struct Opts { #[clap(long, value_name = "INT", global = true)] starfish_rbc_dag_heartbeat_interval_ms: Option, - /// Benchmark-only: sync the framed shadow WAL only at clean shutdown. - /// This removes per-transition disk pressure and is not crash-safe. - #[clap(long, global = true)] - starfish_rbc_dag_shadow_buffered_wal: bool, - /// The type of operation to run. #[clap(subcommand)] operation: Operation, @@ -92,7 +87,6 @@ struct StarfishRbcDagOverrides { shadow: bool, autonomous_clock: bool, heartbeat_interval_ms: Option, - buffered_wal: bool, } /// The type of operation to run. @@ -899,9 +893,6 @@ fn load_benchmark_configs( if let Some(interval_ms) = starfish_rbc_dag.heartbeat_interval_ms { node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; } - if starfish_rbc_dag.buffered_wal { - node_parameters.starfish_rbc_dag_shadow_buffered_wal = true; - } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1086,7 +1077,6 @@ async fn run( shadow: opts.starfish_rbc_dag_shadow, autonomous_clock: opts.starfish_rbc_dag_autonomous_clock, heartbeat_interval_ms: opts.starfish_rbc_dag_heartbeat_interval_ms, - buffered_wal: opts.starfish_rbc_dag_shadow_buffered_wal, }; match opts.operation { Operation::Testbed { action } => match action { @@ -2385,7 +2375,6 @@ mod tests { "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-dag-heartbeat-interval-ms", "125", - "--starfish-rbc-dag-shadow-buffered-wal", "--protocols", "starfish-rbc", ]) @@ -2394,7 +2383,6 @@ mod tests { assert_eq!(opts.block_authentication.as_deref(), Some("mac")); assert!(opts.starfish_rbc_dag_shadow); assert!(opts.starfish_rbc_dag_autonomous_clock); - assert!(opts.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(opts.starfish_rbc_dag_heartbeat_interval_ms, Some(125)); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index e63c0d0f..945900ab 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -286,9 +286,7 @@ impl Measurement { } x if matches!( x.as_str(), - "starfish_rbc_dag_shadow_wal_appended_batches_total" - | "starfish_rbc_dag_shadow_wal_appended_records_total" - | "starfish_rbc_dag_shadow_wal_durable_batches_total" + "starfish_rbc_dag_shadow_wal_durable_batches_total" | "starfish_rbc_dag_shadow_wal_durable_records_total" | "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total" ) => @@ -1024,8 +1022,6 @@ impl MeasurementsCollection { .sum_count_bucket_increments("starfish_rbc_dag_shadow_inputs_total", "delivery,shadow"); let shadow_wal_durable_records = self.sum_scalar_counter_increments("starfish_rbc_dag_shadow_wal_durable_records_total"); - let shadow_wal_appended_records = self - .sum_scalar_counter_increments("starfish_rbc_dag_shadow_wal_appended_records_total"); let shadow_pending_recovery = self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_pending_recovery"); let shadow_unpaired_direct = @@ -1072,7 +1068,7 @@ impl MeasurementsCollection { *scraper_id, "delivery,shadow", ) && self.scalar_counter_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_wal_appended_records_total", + "starfish_rbc_dag_shadow_wal_durable_records_total", *scraper_id, ) && self.latest_scalar_equals( "starfish_rbc_dag_shadow_pending_recovery", @@ -1154,10 +1150,10 @@ impl MeasurementsCollection { *scraper_id, "delivery,shadow", ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_appended_batches_total", + "starfish_rbc_dag_shadow_wal_durable_batches_total", *scraper_id, ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_appended_records_total", + "starfish_rbc_dag_shadow_wal_durable_records_total", *scraper_id, ) && self.scalar_gauge_increased( "starfish_rbc_dag_shadow_carrier_round", @@ -1236,7 +1232,6 @@ impl MeasurementsCollection { shadow_delivery_matches, shadow_delivery_mismatches, shadow_delivery_ambiguous, - shadow_wal_appended_records, shadow_wal_durable_records, shadow_pending_recovery, shadow_unpaired_direct, @@ -1305,7 +1300,7 @@ impl MeasurementsCollection { b->"RBC-DAG shadow:", format!( "valid={} ({}/{} validators), direct={}, shadow={}, matches={}, \ - mismatches={}, ambiguous={}, WAL appended/durable records={}/{}, pending recovery={}, \ + mismatches={}, ambiguous={}, WAL records={}, pending recovery={}, \ unpaired direct/shadow={}/{}, max unpaired lag={} rounds", summary.shadow_comparison_valid, summary.shadow_comparison_valid_nodes, @@ -1315,7 +1310,6 @@ impl MeasurementsCollection { summary.shadow_delivery_matches, summary.shadow_delivery_mismatches, summary.shadow_delivery_ambiguous, - summary.shadow_wal_appended_records, summary.shadow_wal_durable_records, summary.shadow_pending_recovery, summary.shadow_unpaired_direct, @@ -1329,7 +1323,7 @@ impl MeasurementsCollection { b->"RBC-DAG autonomous clock:", format!( "valid={} ({}/{} validators), carrier rounds={}..={}, embedded deliveries={}, phase backlog={}, \ - admitted authors/stake min={}/{}, buffered authenticated={}, WAL appended/durable records={}/{}, \ + admitted authors/stake min={}/{}, buffered authenticated={}, WAL records={}, \ pending recovery={}", summary.shadow_autonomous_clock_valid, summary.shadow_autonomous_clock_valid_nodes, @@ -1341,7 +1335,6 @@ impl MeasurementsCollection { summary.shadow_autonomous_clock_admitted_authors_min, summary.shadow_autonomous_clock_admitted_stake_min, summary.shadow_autonomous_clock_buffered_authenticated_total, - summary.shadow_wal_appended_records, summary.shadow_wal_durable_records, summary.shadow_pending_recovery, ) @@ -1486,15 +1479,6 @@ mod test { ..Measurement::default() }, ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_wal_appended_records_total".to_owned(), - Measurement { - count: wal_durable_records, - scalar: wal_durable_records as f64, - ..Measurement::default() - }, - ); collection.add( scraper_id, "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), @@ -1597,26 +1581,6 @@ mod test { ..Measurement::default() }, ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_wal_appended_batches_total".to_owned(), - Measurement { - timestamp, - count: wal_durable_records, - scalar: wal_durable_records as f64, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_wal_appended_records_total".to_owned(), - Measurement { - timestamp, - count: wal_durable_records, - scalar: wal_durable_records as f64, - ..Measurement::default() - }, - ); collection.add( scraper_id, "starfish_rbc_dag_shadow_wal_durable_batches_total".to_owned(), @@ -1810,8 +1774,6 @@ starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="ambigu # TYPE starfish_rbc_dag_shadow_inputs_total counter starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="shadow"} 7 starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="direct"} 7 -# TYPE starfish_rbc_dag_shadow_wal_appended_records_total counter -starfish_rbc_dag_shadow_wal_appended_records_total{node="node-0"} 42 # TYPE starfish_rbc_dag_shadow_wal_durable_records_total counter starfish_rbc_dag_shadow_wal_durable_records_total{node="node-0"} 42 # TYPE starfish_rbc_dag_shadow_wal_replayed_batches gauge @@ -1839,10 +1801,6 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 measurements["starfish_rbc_dag_shadow_inputs_total"].count_buckets["delivery,shadow"], 7 ); - assert_eq!( - measurements["starfish_rbc_dag_shadow_wal_appended_records_total"].scalar, - 42.0 - ); assert_eq!( measurements["starfish_rbc_dag_shadow_wal_durable_records_total"].scalar, 42.0 @@ -1867,7 +1825,6 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 assert_eq!(summary.shadow_direct_deliveries, 7); assert_eq!(summary.shadow_deliveries, 7); assert_eq!(summary.shadow_delivery_matches, 7); - assert_eq!(summary.shadow_wal_appended_records, 42); assert_eq!(summary.shadow_wal_durable_records, 42); assert_eq!(summary.shadow_unpaired_direct, 2); assert_eq!(summary.shadow_unpaired_shadow, 1); @@ -1939,24 +1896,6 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert!(!summary.shadow_autonomous_clock_valid); } - #[test] - fn autonomous_clock_buffered_wal_uses_appended_progress_without_claiming_durability() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 10, 0, 1, 1, 0, 5); - collection - .data - .remove("starfish_rbc_dag_shadow_wal_durable_batches_total"); - collection - .data - .remove("starfish_rbc_dag_shadow_wal_durable_records_total"); - - let summary = collection.benchmark_run_summary(); - assert!(summary.shadow_autonomous_clock_valid); - assert_eq!(summary.shadow_wal_appended_records, 5); - assert_eq!(summary.shadow_wal_durable_records, 0); - } - #[test] fn missing_final_autonomous_scrape_invalidates_only_clock_verdict() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); @@ -2015,7 +1954,7 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 } #[test] - fn autonomous_clock_valid_gauge_without_wal_progress_is_invalid() { + fn autonomous_clock_valid_gauge_without_durable_progress_is_invalid() { let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 1bb38995..8b3bc56c 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -324,7 +324,6 @@ impl StarfishProtocol { // validator configuration. node_parameters.starfish_rbc_dag_shadow = false; node_parameters.starfish_rbc_dag_autonomous_clock = false; - node_parameters.starfish_rbc_dag_shadow_buffered_wal = false; node_parameters.starfish_rbc_dag_heartbeat_interval_ms = config::node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(); } @@ -371,7 +370,6 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, starfish_rbc_dag_heartbeat_interval_ms: 125, - starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); let parameters = @@ -383,7 +381,6 @@ mod tests { ); assert!(parameters.starfish_rbc_dag_shadow); assert!(parameters.starfish_rbc_dag_autonomous_clock); - assert!(parameters.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } @@ -393,7 +390,6 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, starfish_rbc_dag_heartbeat_interval_ms: 125, - starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }); let parameters = @@ -407,10 +403,6 @@ mod tests { !parameters.starfish_rbc_dag_autonomous_clock, "a global autonomous-clock flag must not leak into non-RBC comparison members" ); - assert!( - !parameters.starfish_rbc_dag_shadow_buffered_wal, - "a global buffered-WAL flag must not leak into non-RBC comparison members" - ); assert_eq!( parameters.starfish_rbc_dag_heartbeat_interval_ms, node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index cf419614..6a455bb4 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -84,12 +84,10 @@ pub struct NodeParameters { /// totality. This must remain an explicit benchmark flag. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, - /// Benchmark-only profile that writes the framed shadow WAL in order but - /// calls `sync_all` only during clean shutdown. This removes persistence - /// pressure from latency experiments and deliberately forfeits the - /// shadow's crash-safety claim for that run. - #[serde(default)] - pub starfish_rbc_dag_shadow_buffered_wal: bool, + /// Maximum interval between autonomous Starfish-RBC-DAG heartbeat + /// carriers. The value is ignored unless the autonomous clock is enabled. + #[serde(default = "node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms")] + pub starfish_rbc_dag_heartbeat_interval_ms: u64, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -172,7 +170,8 @@ impl Default for NodeParameters { starfish_rbc_dag_shadow: false, starfish_rbc_dag_autonomous_clock: false, starfish_rbc_single_dag_echo_qc_fast_path: false, - starfish_rbc_dag_shadow_buffered_wal: false, + starfish_rbc_dag_heartbeat_interval_ms: + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), @@ -415,16 +414,6 @@ impl NodePrivateConfig { self.storage_path .join("starfish-rbc-dag-autonomous-clock-v1.wal") } - - pub fn starfish_rbc_dag_shadow_buffered_benchmark_wal(&self) -> PathBuf { - self.storage_path - .join("starfish-rbc-dag-shadow-buffered-benchmark-v1.wal") - } - - pub fn starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal(&self) -> PathBuf { - self.storage_path - .join("starfish-rbc-dag-autonomous-clock-buffered-benchmark-v1.wal") - } } impl ImportExport for NodePrivateConfig {} @@ -442,7 +431,10 @@ mod tests { assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); - assert!(!parameters.starfish_rbc_dag_shadow_buffered_wal); + assert_eq!( + parameters.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() + ); assert_eq!( parameters.starfish_rbc_dag_heartbeat_interval_ms, node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() @@ -460,7 +452,10 @@ mod tests { assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); - assert!(!decoded.starfish_rbc_dag_shadow_buffered_wal); + assert_eq!( + decoded.starfish_rbc_dag_heartbeat_interval_ms, + node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() + ); assert_eq!( decoded.starfish_rbc_dag_heartbeat_interval_ms, node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() @@ -473,7 +468,6 @@ mod tests { starfish_rbc_dag_shadow: true, starfish_rbc_dag_autonomous_clock: true, starfish_rbc_dag_heartbeat_interval_ms: 125, - starfish_rbc_dag_shadow_buffered_wal: true, ..NodeParameters::default() }; @@ -481,7 +475,6 @@ mod tests { let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); assert!(decoded.starfish_rbc_dag_shadow); assert!(decoded.starfish_rbc_dag_autonomous_clock); - assert!(decoded.starfish_rbc_dag_shadow_buffered_wal); assert_eq!(decoded.starfish_rbc_dag_heartbeat_interval_ms, 125); } @@ -494,14 +487,6 @@ mod tests { private_config.starfish_rbc_dag_shadow_wal(), private_config.starfish_rbc_dag_autonomous_clock_wal() ); - assert_ne!( - private_config.starfish_rbc_dag_autonomous_clock_wal(), - private_config.starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal() - ); - assert_ne!( - private_config.starfish_rbc_dag_shadow_buffered_benchmark_wal(), - private_config.starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal() - ); assert_eq!( private_config.starfish_rbc_dag_autonomous_clock_wal(), Path::new("benchmark") diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index a00075c0..cbb56c7a 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -51,32 +51,6 @@ pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG: i64 = 4; pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR: i64 = 16; pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR: i64 = 2; -const LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES: &[&str] = &[ - "subscribe_broadcast", - "batch", - "missing_parents", - "missing_tx_data", - "partial_sig", - "cert_echo", - "cert_vote", - "cert_ready", - "cert_batch", - "sailfish_timeout", - "sailfish_no_vote", - "unprovable_cert_request", - "round_gap_request", - "rbc_initial", - "rbc_echo", - "rbc_ready", - "rbc_header_request", - "rbc_header_response", - "rbc_dag_shadow_carrier", - "rbc_dag_shadow_carrier_request", - "rbc_dag_shadow_carrier_response", - "rbc_dag_shadow_carrier_sync_request", - "rbc_dag_shadow_carrier_sync_response", -]; - #[derive(Clone)] pub struct Metrics { pub benchmark_duration: IntCounter, @@ -181,8 +155,6 @@ pub struct Metrics { // consensus state. pub starfish_rbc_dag_shadow_inputs_total: IntCounterVec, pub starfish_rbc_dag_shadow_delivery_comparisons_total: IntCounterVec, - pub starfish_rbc_dag_shadow_wal_appended_batches_total: IntCounter, - pub starfish_rbc_dag_shadow_wal_appended_records_total: IntCounter, pub starfish_rbc_dag_shadow_wal_durable_batches_total: IntCounter, pub starfish_rbc_dag_shadow_wal_durable_records_total: IntCounter, pub starfish_rbc_dag_shadow_wal_replayed_batches: IntGauge, @@ -272,18 +244,6 @@ pub struct AutonomousClockBenchmarkBaseline { carrier_round: i64, } -/// Per-validator cumulative counters sampled at the exact start of a local -/// benchmark's active transaction window. Rates subtract this snapshot so -/// connection warmup and shadow-WAL replay are not charged to the protocol. -#[derive(Clone, Debug, Default)] -pub struct LocalBenchmarkCounterBaseline { - sequenced_transactions: u64, - dag_state_entries: u64, - bytes_sent: u64, - bytes_received: u64, - outbound_messages: Vec<(u64, u64)>, -} - #[derive(Debug, Eq, PartialEq)] struct AutonomousClockBenchmarkSummary { valid_nodes: usize, @@ -339,11 +299,11 @@ fn summarize_autonomous_clock_benchmark( .get() > baseline.delivered_carriers && metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total + .starfish_rbc_dag_shadow_wal_durable_batches_total .get() > baseline.wal_batches && metrics - .starfish_rbc_dag_shadow_wal_appended_records_total + .starfish_rbc_dag_shadow_wal_durable_records_total .get() > baseline.wal_records && metrics.starfish_rbc_dag_shadow_carrier_round.get() > baseline.carrier_round @@ -389,7 +349,7 @@ fn summarize_autonomous_clock_benchmark( .iter() .map(|metrics| { metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total + .starfish_rbc_dag_shadow_wal_durable_batches_total .get() }) .sum(); @@ -397,7 +357,7 @@ fn summarize_autonomous_clock_benchmark( .iter() .map(|metrics| { metrics - .starfish_rbc_dag_shadow_wal_appended_records_total + .starfish_rbc_dag_shadow_wal_durable_records_total .get() }) .sum(); @@ -487,38 +447,12 @@ impl Metrics { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .get(), - wal_batches: self - .starfish_rbc_dag_shadow_wal_appended_batches_total - .get(), - wal_records: self - .starfish_rbc_dag_shadow_wal_appended_records_total - .get(), + wal_batches: self.starfish_rbc_dag_shadow_wal_durable_batches_total.get(), + wal_records: self.starfish_rbc_dag_shadow_wal_durable_records_total.get(), carrier_round: self.starfish_rbc_dag_shadow_carrier_round.get(), } } - pub fn local_benchmark_counter_baseline(&self) -> LocalBenchmarkCounterBaseline { - LocalBenchmarkCounterBaseline { - sequenced_transactions: self.sequenced_transactions_total.get(), - dag_state_entries: self.dag_state_entries.get(), - bytes_sent: self.bytes_sent_total.get(), - bytes_received: self.bytes_received_total.get(), - outbound_messages: LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES - .iter() - .map(|request_type| { - ( - self.network_message_bytes_sent_total - .with_label_values(&[request_type]) - .get(), - self.network_requests_sent_total - .with_label_values(&[request_type]) - .get(), - ) - }) - .collect(), - } - } - pub fn new( registry: &Registry, committee: Option<&Committee>, @@ -842,29 +776,15 @@ impl Metrics { registry, ) .unwrap(), - starfish_rbc_dag_shadow_wal_appended_batches_total: - register_int_counter_with_registry!( - "starfish_rbc_dag_shadow_wal_appended_batches_total", - "Starfish-RBC-DAG shadow WAL batches appended to the framed log", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_wal_appended_records_total: - register_int_counter_with_registry!( - "starfish_rbc_dag_shadow_wal_appended_records_total", - "Starfish-RBC-DAG shadow WAL records appended to the framed log", - registry, - ) - .unwrap(), starfish_rbc_dag_shadow_wal_durable_batches_total: register_int_counter_with_registry!( "starfish_rbc_dag_shadow_wal_durable_batches_total", - "Starfish-RBC-DAG shadow WAL batches synchronized before event exposure", + "Starfish-RBC-DAG shadow WAL batches durably synchronized", registry, ) .unwrap(), starfish_rbc_dag_shadow_wal_durable_records_total: register_int_counter_with_registry!( "starfish_rbc_dag_shadow_wal_durable_records_total", - "Starfish-RBC-DAG shadow WAL records synchronized before event exposure", + "Starfish-RBC-DAG shadow WAL records durably synchronized", registry, ) .unwrap(), @@ -1407,69 +1327,32 @@ impl Metrics { starfish_rbc_dag_shadow_expected: bool, starfish_rbc_dag_autonomous_clock_expected: bool, autonomous_clock_baselines: Option>, - counter_baselines: Option>, ) { let num_validators = metrics.len() as u64; // Calculate overall statistics let average_transactions: u64 = metrics .iter() - .enumerate() - .map(|(index, metrics)| { - metrics.sequenced_transactions_total.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.sequenced_transactions) - .unwrap_or_default(), - ) - }) + .map(|m| m.sequenced_transactions_total.get()) .sum::() / num_validators; let average_tps = average_transactions as f64 / duration_secs as f64; let average_blocks_submitted = metrics .iter() - .enumerate() - .map(|(index, metrics)| { - metrics.dag_state_entries.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.dag_state_entries) - .unwrap_or_default(), - ) - }) + .map(|m| m.dag_state_entries.get()) .sum::() / num_validators; let average_bps = average_blocks_submitted as f64 / duration_secs as f64; let average_bytes_sent: u64 = metrics .iter() - .enumerate() - .map(|(index, metrics)| { - metrics.bytes_sent_total.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.bytes_sent) - .unwrap_or_default(), - ) - }) + .map(|m| m.bytes_sent_total.get()) .sum::() / num_validators; let average_bytes_received: u64 = metrics .iter() - .enumerate() - .map(|(index, metrics)| { - metrics.bytes_received_total.get().saturating_sub( - counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(index)) - .map(|baseline| baseline.bytes_received) - .unwrap_or_default(), - ) - }) + .map(|m| m.bytes_received_total.get()) .sum::() / num_validators; let average_reconstructed_sent_to_core: u64 = metrics @@ -1513,21 +1396,6 @@ impl Metrics { .sum::() / num_validators as i64; - // The periodic reporter drains every ten seconds. Pull the final tail - // synchronously so a benchmark cutoff never drops its last samples. - for reporter in &reporters { - reporter - .block_committed_latency - .lock() - .histogram - .receive_all(); - reporter - .transaction_committed_latency - .lock() - .histogram - .receive_all(); - } - let p50_block_committed_latency = reporters .iter() .filter_map(|r| r.block_committed_latency.lock().histogram.pcts([500])) @@ -1577,25 +1445,41 @@ impl Metrics { b->"Average bandwidth in:", format!("{:.2} MB/s", bw_in) ]); - let outbound_message_breakdown = LOCAL_BENCHMARK_NETWORK_MESSAGE_TYPES + const NETWORK_MESSAGE_TYPES: &[&str] = &[ + "subscribe_broadcast", + "batch", + "missing_parents", + "missing_tx_data", + "partial_sig", + "cert_echo", + "cert_vote", + "cert_ready", + "cert_batch", + "sailfish_timeout", + "sailfish_no_vote", + "unprovable_cert_request", + "round_gap_request", + "rbc_initial", + "rbc_echo", + "rbc_ready", + "rbc_header_request", + "rbc_header_response", + "rbc_dag_shadow_carrier", + "rbc_dag_shadow_carrier_request", + "rbc_dag_shadow_carrier_response", + "rbc_dag_shadow_carrier_sync_request", + "rbc_dag_shadow_carrier_sync_response", + ]; + let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() - .enumerate() - .filter_map(|(message_index, request_type)| { + .filter_map(|request_type| { let average_bytes = metrics .iter() - .enumerate() - .map(|(validator_index, metrics)| { - let current = metrics + .map(|metrics| { + metrics .network_message_bytes_sent_total .with_label_values(&[request_type]) - .get(); - let baseline = counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(validator_index)) - .and_then(|baseline| baseline.outbound_messages.get(message_index)) - .map(|(bytes, _)| *bytes) - .unwrap_or_default(); - current.saturating_sub(baseline) + .get() }) .sum::() as f64 / num_validators as f64; @@ -1604,19 +1488,11 @@ impl Metrics { } let average_requests = metrics .iter() - .enumerate() - .map(|(validator_index, metrics)| { - let current = metrics + .map(|metrics| { + metrics .network_requests_sent_total .with_label_values(&[request_type]) - .get(); - let baseline = counter_baselines - .as_ref() - .and_then(|baselines| baselines.get(validator_index)) - .and_then(|baseline| baseline.outbound_messages.get(message_index)) - .map(|(_, requests)| *requests) - .unwrap_or_default(); - current.saturating_sub(baseline) + .get() }) .sum::() as f64 / num_validators as f64; @@ -1679,7 +1555,7 @@ impl Metrics { ) ]); table.add_row(row![ - b->"Clock/WAL progress:", + b->"Durable clock progress:", format!( "heartbeats={}, RBC deliveries={}, WAL batches={}, records={}, open rounds={}..{}", summary.accepted_heartbeats, @@ -2169,12 +2045,6 @@ mod tests { .starfish_rbc_dag_shadow_delivery_comparisons_total .with_label_values(&["match"]) .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_records_total - .inc_by(3); metrics .starfish_rbc_dag_shadow_wal_durable_batches_total .inc(); @@ -2205,8 +2075,6 @@ mod tests { for name in [ "starfish_rbc_dag_shadow_inputs_total", "starfish_rbc_dag_shadow_delivery_comparisons_total", - "starfish_rbc_dag_shadow_wal_appended_batches_total", - "starfish_rbc_dag_shadow_wal_appended_records_total", "starfish_rbc_dag_shadow_wal_durable_batches_total", "starfish_rbc_dag_shadow_wal_durable_records_total", "starfish_rbc_dag_shadow_wal_replayed_batches", @@ -2246,12 +2114,6 @@ mod tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_records_total - .inc_by(2); metrics .starfish_rbc_dag_shadow_wal_durable_batches_total .inc(); @@ -2271,7 +2133,7 @@ mod tests { } #[test] - fn autonomous_clock_summary_requires_every_node_to_make_bounded_wal_progress() { + fn autonomous_clock_summary_requires_every_node_to_make_bounded_durable_progress() { let metrics = vec![ autonomous_clock_metrics(8, 3, 1), autonomous_clock_metrics(9, 4, 2), @@ -2315,12 +2177,6 @@ mod tests { .starfish_rbc_dag_shadow_inputs_total .with_label_values(&["delivery", "shadow"]) .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_appended_records_total - .inc(); metrics .starfish_rbc_dag_shadow_wal_durable_batches_total .inc(); diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index dc3daaf8..effc2889 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -46,10 +46,7 @@ use crate::{ }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, - starfish_rbc_dag::{ - RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId, - storage::ShadowWalSyncPolicyV1, - }, + starfish_rbc_dag::{RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId}, starfish_rbc_dag_shadow::{ ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, }, @@ -1945,11 +1942,6 @@ impl NetworkSyncer metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } let started = if node_parameters.starfish_rbc_dag_autonomous_clock { - let wal_sync_policy = if node_parameters.starfish_rbc_dag_shadow_buffered_wal { - ShadowWalSyncPolicyV1::OnShutdown - } else { - ShadowWalSyncPolicyV1::EveryBatch - }; start_starfish_rbc_dag_autonomous_clock_service_v1( starfish_rbc_dag_shadow_wal, committee_context, @@ -1959,14 +1951,8 @@ impl NetworkSyncer Duration::from_millis( node_parameters.starfish_rbc_dag_heartbeat_interval_ms, ), - wal_sync_policy, ) } else { - let wal_sync_policy = if node_parameters.starfish_rbc_dag_shadow_buffered_wal { - ShadowWalSyncPolicyV1::OnShutdown - } else { - ShadowWalSyncPolicyV1::EveryBatch - }; start_starfish_rbc_dag_shadow_service_v1( starfish_rbc_dag_shadow_wal, committee_context, @@ -1974,7 +1960,6 @@ impl NetworkSyncer context, authorizer, recovered_local_headers, - wal_sync_policy, ) }; match started { @@ -2313,25 +2298,13 @@ impl NetworkSyncer .with_label_values(&[kind, outcome]) .inc(); } - ShadowServiceEventV1::WalAppended { - batches, - records, - durable, - } => { + ShadowServiceEventV1::WalDurable { batches, records } => { shadow_metrics - .starfish_rbc_dag_shadow_wal_appended_batches_total + .starfish_rbc_dag_shadow_wal_durable_batches_total .inc_by(batches); shadow_metrics - .starfish_rbc_dag_shadow_wal_appended_records_total + .starfish_rbc_dag_shadow_wal_durable_records_total .inc_by(records); - if durable { - shadow_metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .inc_by(batches); - shadow_metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .inc_by(records); - } } ShadowServiceEventV1::Ready { autonomous_clock } => { let verdict = if autonomous_clock { diff --git a/crates/starfish-core/src/starfish_rbc_dag/storage.rs b/crates/starfish-core/src/starfish_rbc_dag/storage.rs index 8a1ef827..d553658a 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/storage.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/storage.rs @@ -6,11 +6,9 @@ //! The storage layer deliberately does not encode [`super::journal::JournalEventV1`]. //! A later integration layer owns that versioned codec and the recovery of //! opaque authentication capabilities. This module supplies the durability -//! boundary underneath it: one batch is one checksummed frame. The default -//! [`ShadowWalSyncPolicyV1::EveryBatch`] policy allows a caller to expose the -//! corresponding effects only after [`ShadowWalV1::append_batch`] returns. -//! The explicit benchmark-only `OnShutdown` policy preserves ordered replay -//! on a clean stop but does not provide that crash-durability boundary. +//! boundary underneath it: one batch is one checksummed frame, and a caller +//! may expose the corresponding effects only after [`ShadowWalV1::append_batch`] +//! returns successfully. //! //! Recovery discards only a physically short final frame. A fully present //! frame with a bad header, commit marker, or checksum is reported as @@ -260,18 +258,6 @@ impl From for ShadowWalErrorV1 { } } -/// Persistence boundary used by the non-authoritative shadow WAL. -/// -/// `EveryBatch` is the proof-facing crash-safe mode. `OnShutdown` is an -/// explicit benchmark profile: frames are still written and checksummed in -/// order, but effects may become visible before the kernel has forced those -/// bytes to stable storage. It must never be used to claim crash safety. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ShadowWalSyncPolicyV1 { - EveryBatch, - OnShutdown, -} - pub struct ShadowWalV1 { path: PathBuf, file: File, @@ -279,8 +265,6 @@ pub struct ShadowWalV1 { durable_file_len: u64, batch_count: u64, record_count: u64, - sync_policy: ShadowWalSyncPolicyV1, - batch_sync_count: u64, poisoned: bool, } @@ -294,14 +278,6 @@ impl ShadowWalV1 { pub fn open( path: impl AsRef, namespace: ShadowWalNamespaceV1, - ) -> Result<(Self, ShadowWalRecoveryV1), ShadowWalErrorV1> { - Self::open_with_sync_policy(path, namespace, ShadowWalSyncPolicyV1::EveryBatch) - } - - pub(crate) fn open_with_sync_policy( - path: impl AsRef, - namespace: ShadowWalNamespaceV1, - sync_policy: ShadowWalSyncPolicyV1, ) -> Result<(Self, ShadowWalRecoveryV1), ShadowWalErrorV1> { let path = path.as_ref().to_path_buf(); if let Some(parent) = nonempty_parent(&path) { @@ -344,8 +320,6 @@ impl ShadowWalV1 { durable_file_len: recovery.durable_file_len, batch_count: recovery.batch_count(), record_count: recovery.record_count, - sync_policy, - batch_sync_count: 0, poisoned: false, }; Ok((wal, recovery)) @@ -375,13 +349,7 @@ impl ShadowWalV1 { self.poisoned } - #[cfg(test)] - fn batch_sync_count(&self) -> u64 { - self.batch_sync_count - } - - /// Append one atomic record batch, forcing it to stable storage before - /// returning only under [`ShadowWalSyncPolicyV1::EveryBatch`]. + /// Append and fsync one atomic record batch before returning. /// /// Any seek, write, or fsync failure poisons this handle because the commit /// result may be ambiguous. Drop it and reopen the WAL; recovery will @@ -422,21 +390,15 @@ impl ShadowWalV1 { .checked_add(frame_len) .ok_or(ShadowWalErrorV1::LengthOverflow)?; - let result = self + if let Err(error) = self .file .seek(SeekFrom::Start(start_offset)) .and_then(|_| self.file.write_all(&frame)) - .and_then(|_| match self.sync_policy { - ShadowWalSyncPolicyV1::EveryBatch => self.file.sync_all(), - ShadowWalSyncPolicyV1::OnShutdown => Ok(()), - }); - if let Err(error) = result { + .and_then(|_| self.file.sync_all()) + { self.poisoned = true; return Err(ShadowWalErrorV1::Io(error)); } - if self.sync_policy == ShadowWalSyncPolicyV1::EveryBatch { - self.batch_sync_count = self.batch_sync_count.saturating_add(1); - } self.durable_file_len = end_offset; self.batch_count = next_batch_count; @@ -1400,29 +1362,6 @@ mod tests { wal.shutdown().unwrap(); } - #[test] - fn buffered_benchmark_policy_skips_per_batch_sync_and_reopens_cleanly() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xB1, 0); - let (mut buffered, _) = - ShadowWalV1::open_with_sync_policy(&path, namespace, ShadowWalSyncPolicyV1::OnShutdown) - .unwrap(); - buffered.append_batch(&[b"one".to_vec()]).unwrap(); - buffered.append_batch(&[b"two".to_vec()]).unwrap(); - assert_eq!(buffered.batch_sync_count(), 0); - buffered.shutdown().unwrap(); - - let (mut durable, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!( - recovery.records(), - vec![b"one".as_slice(), b"two".as_slice()] - ); - durable.append_batch(&[b"three".to_vec()]).unwrap(); - assert_eq!(durable.batch_sync_count(), 1); - durable.shutdown().unwrap(); - } - #[test] fn external_file_mutation_poisoning_requires_reopen() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 1fcfd07d..8d62a392 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -29,7 +29,7 @@ use crate::{ model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, storage::{ MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, - ShadowWalSummaryV1, ShadowWalSyncPolicyV1, ShadowWalV1, + ShadowWalSummaryV1, ShadowWalV1, }, }, types::{ @@ -489,35 +489,16 @@ pub(crate) struct StarfishRbcDagShadowV1 { } impl StarfishRbcDagShadowV1 { - #[cfg(test)] pub(crate) fn open( path: impl AsRef, committee: RbcDagCommitteeContextV1, own_authority: AuthorityIndex, context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, - ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { - Self::open_with_wal_sync_policy( - path, - committee, - own_authority, - context, - authorizer, - ShadowWalSyncPolicyV1::EveryBatch, - ) - } - - pub(crate) fn open_with_wal_sync_policy( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { validate_configuration(&committee, own_authority, context, &authorizer)?; let namespace = ShadowWalNamespaceV1::new(context, own_authority); - let (wal, recovery) = ShadowWalV1::open_with_sync_policy(path, namespace, wal_sync_policy)?; + let (wal, recovery) = ShadowWalV1::open(path, namespace)?; let replayed_batches = recovery.batch_count(); let discarded_tail_bytes = recovery.discarded_tail_bytes(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 62501be8..27787191 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -35,7 +35,6 @@ use crate::{ starfish_rbc_dag::{ MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, model::{ModelEffect, ModelError}, - storage::ShadowWalSyncPolicyV1, }, starfish_rbc_dag_shadow::{ ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, @@ -377,10 +376,9 @@ pub(crate) enum ShadowServiceEventV1 { kind: &'static str, outcome: &'static str, }, - WalAppended { + WalDurable { batches: u64, records: u64, - durable: bool, }, Recovered { batches: u64, @@ -566,7 +564,6 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, - wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -583,7 +580,6 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( authorizer, recovered_local_headers, ShadowServiceModeV1::DirectMirror, - wal_sync_policy, ) } @@ -594,7 +590,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( context: RbcDagContextV1, authorizer: ShadowAuthorizerV1, heartbeat_interval: Duration, - wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -614,7 +609,6 @@ pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( authorizer, Vec::new(), ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - wal_sync_policy, ) } @@ -626,7 +620,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( authorizer: ShadowAuthorizerV1, recovered_local_headers: Vec, mode: ShadowServiceModeV1, - wal_sync_policy: ShadowWalSyncPolicyV1, ) -> Result< ( StarfishRbcDagShadowServiceHandleV1, @@ -725,14 +718,7 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); let task = tokio::spawn(async move { let opened = tokio::task::spawn_blocking(move || { - StarfishRbcDagShadowV1::open_with_wal_sync_policy( - path, - committee, - own_authority, - context, - authorizer, - wal_sync_policy, - ) + StarfishRbcDagShadowV1::open(path, committee, own_authority, context, authorizer) }) .await; let (core, open_report) = match opened { @@ -857,7 +843,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let state = ShadowServiceStateV1 { core, mode, - wal_sync_policy, own_authority, committee_size, events: event_tx, @@ -983,7 +968,6 @@ impl ShadowComparisonBacklogV1 { struct ShadowServiceStateV1 { core: StarfishRbcDagShadowV1, mode: ShadowServiceModeV1, - wal_sync_policy: ShadowWalSyncPolicyV1, own_authority: AuthorityIndex, committee_size: usize, events: mpsc::Sender, @@ -1172,11 +1156,7 @@ impl ShadowServiceStateV1 { let batches = after.0.saturating_sub(before.0); let records = after.1.saturating_sub(before.1); if batches != 0 || records != 0 { - self.emit(ShadowServiceEventV1::WalAppended { - batches, - records, - durable: self.wal_sync_policy == ShadowWalSyncPolicyV1::EveryBatch, - }); + self.emit(ShadowServiceEventV1::WalDurable { batches, records }); } } @@ -2065,7 +2045,6 @@ mod tests { self.context, ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), recovered, - ShadowWalSyncPolicyV1::EveryBatch, ) .unwrap() } @@ -2089,23 +2068,6 @@ mod tests { StarfishRbcDagShadowServiceHandleV1, mpsc::Receiver, JoinHandle<()>, - ) { - self.start_autonomous_with_policy( - authority, - heartbeat_interval, - ShadowWalSyncPolicyV1::EveryBatch, - ) - } - - fn start_autonomous_with_policy( - &self, - authority: AuthorityIndex, - heartbeat_interval: Duration, - wal_sync_policy: ShadowWalSyncPolicyV1, - ) -> ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, ) { start_starfish_rbc_dag_autonomous_clock_service_v1( &self.paths[authority as usize], @@ -2114,7 +2076,6 @@ mod tests { self.context, ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), heartbeat_interval, - wal_sync_policy, ) .unwrap() } @@ -2829,51 +2790,6 @@ mod tests { stop(restarted, restarted_events, restarted_task).await; } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn buffered_wal_reports_append_without_durability_and_reopens_after_clean_shutdown() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start_autonomous_with_policy( - 0, - Duration::from_secs(60 * 60), - ShadowWalSyncPolicyV1::OnShutdown, - ); - wait_ready(&mut events).await; - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - - loop { - if let ShadowServiceEventV1::WalAppended { - batches, - records, - durable, - } = next_event(&mut events).await - { - assert_eq!(batches, 1); - assert!(records > 0); - assert!(!durable); - break; - } - } - stop(handle, events, task).await; - - let (restarted, mut restarted_events, restarted_task) = harness.start_autonomous(0); - let mut replayed = 0; - loop { - match next_event(&mut restarted_events).await { - ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches, - ShadowServiceEventV1::Ready { autonomous_clock } => { - assert!(autonomous_clock); - break; - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("buffered WAL restart failed: {error}") - } - _ => {} - } - } - assert_eq!(replayed, 1); - stop(restarted, restarted_events, restarted_task).await; - } - fn phase_carrier( author: AuthorityIndex, statement: RbcPhaseStatementV1, diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 10695e4a..11868ad7 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -91,15 +91,6 @@ impl Validator { "Starfish-RBC-DAG shadow mode requires consensus 'starfish-rbc'" )); } - if public_config - .parameters - .starfish_rbc_dag_shadow_buffered_wal - && !public_config.parameters.starfish_rbc_dag_shadow - { - return Err(eyre!( - "Starfish-RBC-DAG buffered benchmark WAL requires the RBC-DAG shadow" - )); - } if is_starfish_rbc { let protocol_instance = public_config .parameters @@ -243,22 +234,12 @@ impl Validator { } else { None }; - let starfish_rbc_dag_shadow_wal = if public_config - .parameters - .starfish_rbc_dag_shadow_buffered_wal - && public_config.parameters.starfish_rbc_dag_autonomous_clock - { - private_config.starfish_rbc_dag_autonomous_clock_buffered_benchmark_wal() - } else if public_config - .parameters - .starfish_rbc_dag_shadow_buffered_wal - { - private_config.starfish_rbc_dag_shadow_buffered_benchmark_wal() - } else if public_config.parameters.starfish_rbc_dag_autonomous_clock { - private_config.starfish_rbc_dag_autonomous_clock_wal() - } else { - private_config.starfish_rbc_dag_shadow_wal() - }; + let starfish_rbc_dag_shadow_wal = + if public_config.parameters.starfish_rbc_dag_autonomous_clock { + private_config.starfish_rbc_dag_autonomous_clock_wal() + } else { + private_config.starfish_rbc_dag_shadow_wal() + }; let (core, bls_cert_aggregator) = Core::open( block_handler, @@ -443,36 +424,6 @@ mod smoke_tests { })); } - #[tokio::test] - async fn buffered_shadow_wal_requires_shadow_mode() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config - .parameters - .starfish_rbc_dag_shadow_buffered_wal = true; - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("buffered benchmark WAL requires the RBC-DAG shadow") - })); - } - #[tokio::test] async fn autonomous_clock_rejects_non_rbc_protocol() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 236cdb06..39344955 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -196,10 +196,9 @@ enum Operation { /// benchmark runs. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, - /// Benchmark-only: write ordered shadow-WAL frames but force them to - /// stable storage only at clean shutdown. This run is not crash-safe. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_shadow_buffered_wal: bool, + /// Maximum interval between autonomous RBC-DAG heartbeat carriers. + #[clap(long, value_name = "INT")] + starfish_rbc_dag_heartbeat_interval_ms: Option, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, /// Dissemination mode override: @@ -313,7 +312,7 @@ async fn main() -> Result<()> { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_single_dag_echo_qc_fast_path, - starfish_rbc_dag_shadow_buffered_wal, + starfish_rbc_dag_heartbeat_interval_ms, duration_secs, dissemination_mode, } => { @@ -328,8 +327,9 @@ async fn main() -> Result<()> { node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = starfish_rbc_single_dag_echo_qc_fast_path; - node_parameters.starfish_rbc_dag_shadow_buffered_wal = - starfish_rbc_dag_shadow_buffered_wal; + if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { + node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; + } if is_starfish_rbc_selection(&consensus_protocol) { node_parameters.refresh_starfish_rbc_protocol_instance(); } @@ -420,10 +420,6 @@ async fn local_benchmark( consensus_protocol: String, duration_secs: u64, ) -> Result<()> { - eyre::ensure!( - duration_secs > 0, - "benchmark duration must be greater than zero" - ); println!("\n=== Benchmark Configuration ==="); println!("Committee Size: {committee_size}"); println!("Byzantine Nodes: {num_byzantine_nodes}"); @@ -443,16 +439,6 @@ async fn local_benchmark( .unwrap_or("ed25519") } ); - if node_parameters.starfish_rbc_dag_shadow { - println!( - "Shadow WAL: {}", - if node_parameters.starfish_rbc_dag_shadow_buffered_wal { - "buffered benchmark profile (sync on clean shutdown; not crash-safe)" - } else { - "sync every transition (crash-safe reference profile)" - } - ); - } if node_parameters.starfish_rbc_single_dag_echo_qc_fast_path { println!( "Single-DAG receiver-local quorum-ECHO: ENABLED (signature-free latency lower bound; Byzantine totality not provided)" @@ -483,15 +469,15 @@ async fn local_benchmark( let ips = vec![IpAddr::V4(Ipv4Addr::LOCALHOST); committee_size]; let committee = Committee::new_for_benchmarks(committee_size); load /= committee.len(); - let mut parameters = Parameters::almost_default(load); - parameters.benchmark_duration = Some(Duration::from_secs(duration_secs)); + let parameters = Parameters::almost_default(load); // Equivocating Byzantine strategies must not generate transactions. - let mut byzantine_parameters = parameters.clone(); - if ByzantineStrategy::from_strategy_str(&byzantine_strategy) + let byzantine_parameters = if ByzantineStrategy::from_strategy_str(&byzantine_strategy) .is_some_and(|s| s.is_equivocating()) { - byzantine_parameters.load = 0; - } + Parameters::almost_default(0) + } else { + parameters.clone() + }; let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; let starfish_rbc_dag_autonomous_clock_expected = @@ -622,36 +608,12 @@ async fn local_benchmark( } } - // `duration_secs` is an active transaction-submission window, not a - // process-lifetime cutoff. Every finite generator holds metrics inactive - // through connection warmup, then opens this latch immediately before its - // first batch. Start the benchmark only after every honest validator has - // crossed that boundary. - tokio::time::timeout(Duration::from_secs(30), async { - loop { - if metrics_of_honest_validators - .iter() - .all(|metrics| metrics.metrics_active.load(Ordering::Relaxed)) - { - break; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .wrap_err("transaction generators did not open the active benchmark window")?; - println!("Active transaction window started ({duration_secs} seconds)"); - let autonomous_clock_baselines = starfish_rbc_dag_autonomous_clock_expected.then(|| { metrics_of_honest_validators .iter() .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) .collect::>() }); - let counter_baselines = metrics_of_honest_validators - .iter() - .map(|metrics| metrics.local_benchmark_counter_baseline()) - .collect::>(); // Run for specified duration tokio::select! { @@ -669,7 +631,6 @@ async fn local_benchmark( starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, autonomous_clock_baselines.clone(), - Some(counter_baselines.clone()), ); // Abort all tasks @@ -697,7 +658,6 @@ async fn local_benchmark( starfish_rbc_dag_shadow_expected, starfish_rbc_dag_autonomous_clock_expected, autonomous_clock_baselines, - Some(counter_baselines), ); fs::remove_dir_all(base_dir)?; eyre::bail!("All validators completed before the requested benchmark duration") @@ -1067,7 +1027,8 @@ mod tests { "--starfish-rbc-dag-shadow", "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-single-dag-echo-qc-fast-path", - "--starfish-rbc-dag-shadow-buffered-wal", + "--starfish-rbc-dag-heartbeat-interval-ms", + "125", ]) .unwrap(); @@ -1077,7 +1038,7 @@ mod tests { starfish_rbc_dag_shadow, starfish_rbc_dag_autonomous_clock, starfish_rbc_single_dag_echo_qc_fast_path, - starfish_rbc_dag_shadow_buffered_wal, + starfish_rbc_dag_heartbeat_interval_ms, .. } = args.operation else { @@ -1088,7 +1049,7 @@ mod tests { assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_single_dag_echo_qc_fast_path); - assert!(starfish_rbc_dag_shadow_buffered_wal); + assert_eq!(starfish_rbc_dag_heartbeat_interval_ms, Some(125)); } #[test] diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 15280547..ae5da439 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -14,9 +14,7 @@ The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That s implemented. The milestone-three direct-header comparison runtime is enabled with `--consensus starfish-rbc --starfish-rbc-dag-shadow`. Milestone four adds a separate control-only runtime with `--starfish-rbc-dag-autonomous-clock`; its carrier rounds advance independently through -authenticated admission and empty heartbeats. Performance experiments may add -`--starfish-rbc-dag-shadow-buffered-wal` to remove per-transition disk synchronization; that -profile is explicitly not crash-safe. Both modes leave the direct prototype's DAG, +authenticated admission and empty heartbeats. Both modes leave the direct prototype's DAG, pacemaker, commit, and output unchanged. The eventual protocol is new, not a transport option or a version-two alias for `starfish-rbc`. @@ -90,15 +88,13 @@ handle even after its async supervisor is detached. A same-process restart again therefore forbidden until the worker has exited; process exit remains safe. A production-quality same-process restart path needs an operating-system file lock or a fully cancellable storage task. -The shadow runtime is not a proof or a production performance implementation. Its default -crash-safe profile intentionally fsyncs every accepted transition, and its reference reducer clones -retained model/journal history. A separate explicit benchmark profile writes the same ordered, -checksummed frames but syncs them only on clean shutdown; it reports appended and durable records -separately and makes no crash-safety claim. This removes the known persistence observer effect -without changing the protocol reducer. The runtime also uses a fixed unsolicited-retention window -only as a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the -composition and resource bounds are completed, `starfish-rbc-dag` remains an experimental -shadow/reference implementation rather than a proven signature-free Starfish variant. +The shadow runtime is not a proof or a performance implementation. It intentionally fsyncs every +accepted transition and its reference reducer clones retained model/journal history, so total CPU +work grows superlinearly with a long run. It also uses a fixed unsolicited-retention window only as +a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the +composition, resource bounds, and performance path are completed, `starfish-rbc-dag` must be +described as an experimental shadow/reference implementation rather than a proven signature-free +Starfish variant or a fair throughput baseline. The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not @@ -789,10 +785,9 @@ newest current-process observation (`<= 4`). These are empirical benchmark cover asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as proof of a protocol failure. -The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a slow -reference transition (including synchronous fsync in the crash-safe profile), capping queued -maximum-sized carrier bodies at 256 MiB (plus sidecars and allocator overhead). Mirror mode budgets -one peer fan-in plus five local/control inputs and accepts +The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a +synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars and +allocator overhead). Mirror mode budgets one peer fan-in plus five local/control inputs and accepts at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, @@ -801,7 +796,7 @@ per peer. Requested historical slots remain recoverable beyond the benchmark-onl retention window. Autonomous benchmark validity is separate from delivery comparison validity. -`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the appended-WAL and heartbeat counters must progress, +`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the WAL and heartbeat counters must progress, the carrier round and embedded-RBC delivery count must advance during the measured interval, recovery must drain, and the reported clock-state/backlog and cross-node skew must remain within the configured empirical guards. These checks establish that the observational carrier plane stayed @@ -907,32 +902,20 @@ state. Batching can reduce the number of separately scheduled RBC control messages, but it does not remove their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout -transport is added. Shadow mode also sends both direct and embedded transcripts. The default -crash-safe reference profile fsyncs each accepted transition and validates through a clone-based -reducer; it is a correctness/replay instrument, not a protocol-performance result. The explicit -buffered-WAL profile keeps the exact framed event path but syncs only on clean shutdown and therefore -cannot be used for crash-safety claims. Benchmark output reports appended and durable WAL work -separately. The clone-based reducer remains intentionally unoptimized until measurement shows it -matters. - -A matched 10-validator local A/B on 2026-08-11 used a full 60-second active transaction window, -the AWS RTT emulator, nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous -heartbeat. The harness waits through generator warmup, snapshots cumulative counters at the active -boundary, and drains final latency samples. - -| Profile | Verdict | TPS | p50 block | p50 E2E | Outbound | -|---|---:|---:|---:|---:|---:| -| Direct Starfish-RBC, shadow off | n/a | 972.25 | 1,508.0 ms | 1,724.0 ms | 0.53 MB/s | -| Autonomous RBC-DAG, buffered WAL | VALID 10/10 | 971.37 | 1,498.9 ms | 1,714.0 ms | 0.58 MB/s | -| Autonomous RBC-DAG, per-transition fsync | INVALID 9/10 | 948.83 | 2,569.1 ms | 3,067.4 ms | 0.54 MB/s | - -The valid buffered run reached carrier round 275 at every validator, with 2,749 accepted -heartbeats, 27,180 embedded-RBC deliveries, 27,424 appended batches, zero round skew, and zero -pending recovery. Its latency and throughput match the shadow-off baseline while making the -expected extra carrier traffic visible. The crash-safe run shed shadow work and is reported only -as a diagnostic: it isolates synchronous persistence as a severe observer effect and must not be -cited as a protocol result. Neither run measures application latency through the carrier DAG yet, -because the direct Starfish-RBC path remains authoritative in milestone four. +transport is added. Shadow mode also sends both direct and embedded transcripts and is a correctness +instrument, not a performance result. In milestone three it additionally fsyncs each accepted +transition and validates through a clone-based reference reducer. Those costs are deliberately not +charged as protocol overhead: performance runs require incremental state transitions/checkpoints +or an equivalently durable baseline, plus separate WAL/fsync accounting. + +As an implementation-continuity check, a 10-validator, 60-second local run on 2026-08-11 used the +AWS RTT emulator, a nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. +All validators reached carrier round 196 with zero skew and zero pending recovery; the control +plane recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative direct +Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 end-to-end +latency, and 0.45 MB/s average outbound bandwidth. This is not a comparative performance claim: +the cutoff includes the local generator warmup, and the control-only shadow still performs +per-transition fsync/reference-model work on the authoritative network socket. ## 19. Contained implementation milestones From 8a483ccbff01de084761ea72546e92d20fd6e183 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:45:22 +0200 Subject: [PATCH 58/62] Revert "Add autonomous Starfish-RBC-DAG shadow clock" This reverts commit 68a0a68c74ee149cb62504dcf8ce4856c402d537. --- README.md | 44 +- crates/orchestrator/src/benchmark.rs | 91 +- crates/orchestrator/src/main.rs | 50 +- crates/orchestrator/src/measurements.rs | 550 +------ crates/orchestrator/src/orchestrator.rs | 16 +- crates/orchestrator/src/protocol/starfish.rs | 49 +- crates/starfish-core/src/config.rs | 76 +- crates/starfish-core/src/metrics.rs | 457 +----- crates/starfish-core/src/net_sync.rs | 203 +-- crates/starfish-core/src/network.rs | 73 - .../src/starfish_rbc_dag_shadow.rs | 224 +-- .../src/starfish_rbc_dag_shadow_service.rs | 1371 +---------------- crates/starfish-core/src/validator.rs | 185 +-- crates/starfish/src/main.rs | 105 +- docs/starfish-rbc-dag-protocol.md | 98 +- 15 files changed, 195 insertions(+), 3397 deletions(-) diff --git a/README.md b/README.md index 8d6c8ded..7445b861 100644 --- a/README.md +++ b/README.md @@ -50,41 +50,25 @@ Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctnes prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). **Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate logical projection. Its -canonical types, deterministic models, crash journal, direct-header comparison shadow, and a -separate opt-in autonomous heartbeat carrier clock are implemented. Run the comparison shadow with -`--consensus starfish-rbc --starfish-rbc-dag-shadow`. Add -`--starfish-rbc-dag-autonomous-clock` to run an independent control-only carrier clock (prototype -heartbeat default: 250 ms); that mode deliberately does not map direct application headers to -carrier rounds or claim a direct-delivery comparison. Direct Starfish-RBC remains solely -authoritative in both modes, and shadow failures or results cannot affect proposals, commits, or -output as protocol state. Shadow traffic still shares the validator's network socket and bandwidth, -so it can perturb timing, and it must be enabled only on a homogeneous new-binary committee; there -is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` selector is not -implemented yet. The shadow uses per-transition fsync and a clone-based reference reducer, so it is -a correctness instrument, not a fair performance baseline, and carries no safety or liveness claim. -Its WAL can reopen the shadow actor, but this is not full validator crash recovery: authoritative -direct Starfish-RBC phase and delivery locks are not durable yet, so that baseline remains fail-stop -across process restart. The design and proof obligations are documented in the +canonical types, deterministic models, crash journal, and an opt-in persisted network shadow are +implemented. Run the shadow with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; direct +Starfish-RBC remains solely authoritative and shadow failures or results cannot affect proposals, +commits, or output as protocol state. Shadow traffic still shares the validator's network socket and +bandwidth, so it can perturb timing, and it must be enabled only on a homogeneous new-binary +committee; there is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` +selector is not implemented yet. The shadow uses per-transition fsync and a clone-based reference +reducer, so it is a correctness instrument, not a fair performance baseline, and carries no safety +or liveness claim. Its WAL can reopen the shadow actor against matching recovered direct headers, +but this is not full validator crash recovery: authoritative direct Starfish-RBC phase and delivery +locks are not durable yet, so that baseline remains fail-stop across process restart. The design and +proof obligations are documented in the [protocol design](docs/starfish-rbc-dag-protocol.md). -For a direct-header shadow comparison, +For any shadow comparison, `starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded observational path was disabled or shed work and the comparison must be discarded. Healthy live production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between -the cumulative direct and shadow delivery counters. Autonomous runs instead require -`starfish_rbc_dag_shadow_clock_valid == 1`, durable heartbeat progress, advancing carrier rounds, -in-window embedded-RBC delivery, and bounded clock-state gauges. The current queue budget supports -at most 60 validators in mirror mode and 20 in autonomous mode. - -An exploratory 10-validator, 60-second local run with the AWS RTT emulator, nominal 1,000 tx/s -load, MAC authentication, and a 250 ms autonomous heartbeat completed with a `VALID` clock -verdict on 2026-08-11. All validators reached carrier round 196 with zero skew and zero pending -recovery; the run recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative -direct Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 -end-to-end latency, and 0.45 MB/s average outbound bandwidth. This is a prototype continuity -result, not a fair performance comparison: the 60-second cutoff includes the local generator -warmup, and the control-only shadow still performs per-transition fsync/reference-model work while -sharing the authoritative socket. +the cumulative direct and shadow delivery counters. **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index f29acaf9..dc024546 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -125,31 +125,6 @@ pub struct BenchmarkRunSummary { pub shadow_unpaired_shadow: usize, #[serde(default)] pub shadow_unpaired_max_round_lag: usize, - /// Whether this run used the independent, non-authoritative carrier clock - /// instead of the direct-RBC mirror comparison. - #[serde(default)] - pub shadow_autonomous_clock_enabled: bool, - /// Sticky run verdict: every validator that was ready at benchmark start - /// exposed `clock_valid == 1`, made active-window heartbeat, embedded-RBC - /// delivery, WAL, and carrier-round progress, stayed within the - /// experimental live-state/skew bounds, and supplied a valid mandatory - /// final scrape. - #[serde(default)] - pub shadow_autonomous_clock_valid: bool, - #[serde(default)] - pub shadow_autonomous_clock_valid_nodes: usize, - #[serde(default)] - pub shadow_autonomous_clock_carrier_round_min: usize, - #[serde(default)] - pub shadow_autonomous_clock_carrier_round_max: usize, - #[serde(default)] - pub shadow_autonomous_clock_phase_backlog_total: usize, - #[serde(default)] - pub shadow_autonomous_clock_admitted_authors_min: usize, - #[serde(default)] - pub shadow_autonomous_clock_admitted_stake_min: usize, - #[serde(default)] - pub shadow_autonomous_clock_buffered_authenticated_total: usize, } impl BenchmarkRunSummary { @@ -175,15 +150,7 @@ impl BenchmarkRunSummary { shadow_delivery_mismatches,shadow_delivery_ambiguous,\ shadow_wal_durable_records,shadow_pending_recovery,\ shadow_unpaired_direct,shadow_unpaired_shadow,\ - shadow_unpaired_max_round_lag,\ - shadow_autonomous_clock_enabled,shadow_autonomous_clock_valid,\ - shadow_autonomous_clock_valid_nodes,\ - shadow_autonomous_clock_carrier_round_min,\ - shadow_autonomous_clock_carrier_round_max,\ - shadow_autonomous_clock_phase_backlog_total,\ - shadow_autonomous_clock_admitted_authors_min,\ - shadow_autonomous_clock_admitted_stake_min,\ - shadow_autonomous_clock_buffered_authenticated_total" + shadow_unpaired_max_round_lag" } pub fn csv_record(&self) -> String { @@ -230,17 +197,6 @@ impl BenchmarkRunSummary { self.shadow_unpaired_direct.to_string(), self.shadow_unpaired_shadow.to_string(), self.shadow_unpaired_max_round_lag.to_string(), - self.shadow_autonomous_clock_enabled.to_string(), - self.shadow_autonomous_clock_valid.to_string(), - self.shadow_autonomous_clock_valid_nodes.to_string(), - self.shadow_autonomous_clock_carrier_round_min.to_string(), - self.shadow_autonomous_clock_carrier_round_max.to_string(), - self.shadow_autonomous_clock_phase_backlog_total.to_string(), - self.shadow_autonomous_clock_admitted_authors_min - .to_string(), - self.shadow_autonomous_clock_admitted_stake_min.to_string(), - self.shadow_autonomous_clock_buffered_authenticated_total - .to_string(), ] .join(",") } @@ -821,8 +777,8 @@ pub mod test { use crate::settings::Settings; use super::{ - BenchmarkParametersGeneric, BenchmarkRunSummary, CommitteeScalingPlan, - LatencyThroughputSweepPlan, ProtocolParameters, StabilityOutage, + BenchmarkParametersGeneric, CommitteeScalingPlan, LatencyThroughputSweepPlan, + ProtocolParameters, StabilityOutage, }; /// Mock benchmark type for unit tests. @@ -850,47 +806,6 @@ pub mod test { type TestBenchmarkParameters = BenchmarkParametersGeneric; - #[test] - fn benchmark_csv_includes_autonomous_clock_verdict_and_state() { - let summary = BenchmarkRunSummary { - shadow_autonomous_clock_enabled: true, - shadow_autonomous_clock_valid: true, - shadow_autonomous_clock_valid_nodes: 4, - shadow_autonomous_clock_carrier_round_min: 10, - shadow_autonomous_clock_carrier_round_max: 12, - shadow_autonomous_clock_phase_backlog_total: 3, - shadow_autonomous_clock_admitted_authors_min: 3, - shadow_autonomous_clock_admitted_stake_min: 7, - shadow_autonomous_clock_buffered_authenticated_total: 2, - ..BenchmarkRunSummary::default() - }; - let headers = BenchmarkRunSummary::csv_header() - .split(',') - .map(str::trim) - .collect::>(); - let record = summary.csv_record(); - let values = record.split(',').collect::>(); - assert_eq!(headers.len(), values.len()); - - for (header, expected) in [ - ("shadow_autonomous_clock_enabled", "true"), - ("shadow_autonomous_clock_valid", "true"), - ("shadow_autonomous_clock_valid_nodes", "4"), - ("shadow_autonomous_clock_carrier_round_min", "10"), - ("shadow_autonomous_clock_carrier_round_max", "12"), - ("shadow_autonomous_clock_phase_backlog_total", "3"), - ("shadow_autonomous_clock_admitted_authors_min", "3"), - ("shadow_autonomous_clock_admitted_stake_min", "7"), - ("shadow_autonomous_clock_buffered_authenticated_total", "2"), - ] { - let index = headers - .iter() - .position(|candidate| *candidate == header) - .unwrap(); - assert_eq!(values[index], expected, "column {header}"); - } - } - #[test] fn latency_throughput_sweep_switches_to_fine_grained_steps() { let plan = LatencyThroughputSweepPlan::new( diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 1c30ea52..eff22b62 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -63,32 +63,15 @@ pub struct Opts { #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, - /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative - /// shadow. + /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative shadow. #[clap(long, global = true)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. - #[clap(long, global = true)] - starfish_rbc_dag_autonomous_clock: bool, - - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT", global = true)] - starfish_rbc_dag_heartbeat_interval_ms: Option, - /// The type of operation to run. #[clap(subcommand)] operation: Operation, } -#[derive(Clone, Copy, Debug)] -struct StarfishRbcDagOverrides { - shadow: bool, - autonomous_clock: bool, - heartbeat_interval_ms: Option, -} - /// The type of operation to run. #[derive(Parser, Debug)] #[clap(rename_all = "kebab-case")] @@ -873,7 +856,7 @@ fn load_benchmark_configs( compress_network: Option, bls_workers: Option, block_authentication: &Option, - starfish_rbc_dag: StarfishRbcDagOverrides, + starfish_rbc_dag_shadow: bool, ) -> eyre::Result<(NodeParameters, ClientParameters)> { let mut node_parameters = match &settings.node_parameters_path { Some(path) => NodeParameters::load(path).wrap_err("Failed to load node's parameters")?, @@ -884,15 +867,9 @@ fn load_benchmark_configs( if block_authentication.is_some() { node_parameters.block_authentication = block_authentication.clone(); } - if starfish_rbc_dag.shadow { + if starfish_rbc_dag_shadow { node_parameters.starfish_rbc_dag_shadow = true; } - if starfish_rbc_dag.autonomous_clock { - node_parameters.starfish_rbc_dag_autonomous_clock = true; - } - if let Some(interval_ms) = starfish_rbc_dag.heartbeat_interval_ms { - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; - } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1073,11 +1050,7 @@ async fn run( .wrap_err("Failed to crate testbed")?; let block_authentication = opts.block_authentication.clone(); - let starfish_rbc_dag = StarfishRbcDagOverrides { - shadow: opts.starfish_rbc_dag_shadow, - autonomous_clock: opts.starfish_rbc_dag_autonomous_clock, - heartbeat_interval_ms: opts.starfish_rbc_dag_heartbeat_interval_ms, - }; + let starfish_rbc_dag_shadow = opts.starfish_rbc_dag_shadow; match opts.operation { Operation::Testbed { action } => match action { // Display the current status of the testbed. @@ -1275,7 +1248,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1443,7 +1416,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1651,7 +1624,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -1818,7 +1791,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -2025,7 +1998,7 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag, + starfish_rbc_dag_shadow, )?; display::newline(); @@ -2372,9 +2345,6 @@ mod tests { "--block-authentication", "mac", "--starfish-rbc-dag-shadow", - "--starfish-rbc-dag-autonomous-clock", - "--starfish-rbc-dag-heartbeat-interval-ms", - "125", "--protocols", "starfish-rbc", ]) @@ -2382,8 +2352,6 @@ mod tests { assert_eq!(opts.block_authentication.as_deref(), Some("mac")); assert!(opts.starfish_rbc_dag_shadow); - assert!(opts.starfish_rbc_dag_autonomous_clock); - assert_eq!(opts.starfish_rbc_dag_heartbeat_interval_ms, Some(125)); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); }; diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index 945900ab..d4a78d4d 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -16,10 +16,7 @@ use prettytable::{Table, row}; use prometheus_parse::Scrape; use serde::{Deserialize, Serialize}; use starfish_core::metrics::{ - STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR, - STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR, - STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, - STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, + STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, }; use crate::{ @@ -33,28 +30,6 @@ type BucketId = String; /// The identifier of a measurement type. type Label = String; -pub(crate) const SHADOW_COMPARISON_VALID_METRIC: &str = "starfish_rbc_dag_shadow_comparison_valid"; -pub(crate) const SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC: &str = "starfish_rbc_dag_shadow_clock_valid"; - -/// Select the validity gauge for the configured observational mode. Mirror -/// mode compares direct and embedded RBC deliveries; autonomous mode has no -/// one-to-one direct stream and therefore owns a separate clock verdict. -pub(crate) fn shadow_validity_metric(parameters: &BenchmarkParameters) -> Option<&'static str> { - if parameters.consensus_protocol != "starfish-rbc" - || !parameters.node_parameters.starfish_rbc_dag_shadow - { - return None; - } - - Some( - if parameters.node_parameters.starfish_rbc_dag_autonomous_clock { - SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC - } else { - SHADOW_COMPARISON_VALID_METRIC - }, - ) -} - /// A snapshot measurement at a given time. #[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq)] pub struct Measurement { @@ -304,12 +279,6 @@ impl Measurement { "starfish_rbc_dag_shadow_wal_replayed_batches" | "starfish_rbc_dag_shadow_pending_recovery" | "starfish_rbc_dag_shadow_comparison_valid" - | "starfish_rbc_dag_shadow_clock_valid" - | "starfish_rbc_dag_shadow_carrier_round" - | "starfish_rbc_dag_shadow_phase_backlog" - | "starfish_rbc_dag_shadow_admitted_authors" - | "starfish_rbc_dag_shadow_admitted_stake" - | "starfish_rbc_dag_shadow_buffered_authenticated" | "starfish_rbc_dag_shadow_unpaired_direct" | "starfish_rbc_dag_shadow_unpaired_shadow" | "starfish_rbc_dag_shadow_unpaired_max_round_lag" @@ -443,9 +412,6 @@ impl MeasurementsCollection { /// shadow scrape. Appending an explicit invalid observation prevents an /// earlier successful scrape from being mistaken for fresh final evidence. pub fn mark_shadow_final_scrape_missing(&mut self, scraper_id: ScraperId) { - let Some(validity_metric) = shadow_validity_metric(&self.parameters) else { - return; - }; let timestamp = self .data .values() @@ -458,7 +424,7 @@ impl MeasurementsCollection { self.synthetic_only_scrapers.insert(scraper_id); } self.data - .entry(validity_metric.to_owned()) + .entry("starfish_rbc_dag_shadow_comparison_valid".to_owned()) .or_default() .entry(scraper_id) .or_default() @@ -563,14 +529,6 @@ impl MeasurementsCollection { .sum() } - fn min_latest_scalar_as_usize(&self, label: &str) -> usize { - self.latest_measurements(label) - .into_iter() - .map(|measurement| measurement.scalar.max(0.0) as usize) - .min() - .unwrap_or_default() - } - /// Sum scalar Prometheus counter increments across scrapers and resets. fn sum_scalar_counter_increments(&self, label: &str) -> usize { self.data @@ -598,27 +556,6 @@ impl MeasurementsCollection { self.data.get(label)?.get(&scraper_id).map(Vec::as_slice) } - fn active_window_series(&self, label: &str, scraper_id: ScraperId) -> Option<&[Measurement]> { - let series = self.scraper_series(label, scraper_id)?; - if !series - .windows(2) - .all(|window| window[1].timestamp >= window[0].timestamp) - { - return None; - } - let start = series - .iter() - .position(|measurement| !measurement.timestamp.is_zero())?; - let maximum_timestamp = series.iter().map(Measurement::timestamp).max()?; - let end = series - .iter() - .position(|measurement| measurement.timestamp == maximum_timestamp)?; - let active = series.get(start..=end)?; - let first = active.first()?; - let last = active.last()?; - (active.len() >= 2 && last.timestamp > first.timestamp).then_some(active) - } - fn gauge_always_equals(&self, label: &str, scraper_id: ScraperId, expected: f64) -> bool { self.scraper_series(label, scraper_id) .is_some_and(|series| { @@ -642,20 +579,6 @@ impl MeasurementsCollection { .is_some_and(|measurement| measurement.scalar > 0.0) } - fn scalar_counter_increased(&self, label: &str, scraper_id: ScraperId) -> bool { - let Some(series) = self.active_window_series(label, scraper_id) else { - return false; - }; - series.len() >= 2 - && series - .windows(2) - .all(|window| window[1].scalar >= window[0].scalar) - && series - .last() - .zip(series.first()) - .is_some_and(|(last, first)| last.scalar > first.scalar) - } - fn count_bucket_is_monotonic_and_positive( &self, label: &str, @@ -673,22 +596,6 @@ impl MeasurementsCollection { && values.last().is_some_and(|value| *value > 0) } - fn count_bucket_increased(&self, label: &str, scraper_id: ScraperId, bucket: &str) -> bool { - let Some(series) = self.active_window_series(label, scraper_id) else { - return false; - }; - let values = series - .iter() - .map(|measurement| measurement.count_buckets.get(bucket).copied().unwrap_or(0)) - .collect::>(); - values.len() >= 2 - && values.windows(2).all(|window| window[1] >= window[0]) - && values - .last() - .zip(values.first()) - .is_some_and(|(last, first)| last > first) - } - fn count_bucket_is_always_zero( &self, label: &str, @@ -708,26 +615,6 @@ impl MeasurementsCollection { .is_some_and(|measurement| measurement.scalar == expected) } - fn latest_scalar_greater_than(&self, label: &str, scraper_id: ScraperId, minimum: f64) -> bool { - self.scraper_series(label, scraper_id) - .and_then(|series| series.last()) - .is_some_and(|measurement| measurement.scalar > minimum) - } - - fn scalar_gauge_increased(&self, label: &str, scraper_id: ScraperId) -> bool { - self.active_window_series(label, scraper_id) - .is_some_and(|series| { - series.len() >= 2 - && series - .windows(2) - .all(|window| window[1].scalar >= window[0].scalar) - && series - .last() - .zip(series.first()) - .is_some_and(|(last, first)| last.scalar > first.scalar) - }) - } - fn gauge_always_at_most(&self, label: &str, scraper_id: ScraperId, maximum: f64) -> bool { self.scraper_series(label, scraper_id) .is_some_and(|series| { @@ -973,14 +860,8 @@ impl MeasurementsCollection { } }) .collect(); - let shadow_enabled = self.parameters.consensus_protocol == "starfish-rbc" + let shadow_comparison_enabled = self.parameters.consensus_protocol == "starfish-rbc" && self.parameters.node_parameters.starfish_rbc_dag_shadow; - let shadow_autonomous_clock_enabled = shadow_enabled - && self - .parameters - .node_parameters - .starfish_rbc_dag_autonomous_clock; - let shadow_comparison_enabled = shadow_enabled && !shadow_autonomous_clock_enabled; let shadow_valid_scrapers = self .data .get("starfish_rbc_dag_shadow_comparison_valid") @@ -1094,103 +975,6 @@ impl MeasurementsCollection { && every_shadow_scraper_has_coverage && shadow_delivery_mismatches == 0 && shadow_delivery_ambiguous == 0; - let shadow_autonomous_clock_valid_scrapers = self - .data - .get(SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC) - .map(|by_scraper| { - by_scraper - .keys() - .copied() - .filter(|scraper_id| { - self.gauge_always_equals( - SHADOW_AUTONOMOUS_CLOCK_VALID_METRIC, - *scraper_id, - 1.0, - ) - }) - .collect::>() - }) - .unwrap_or_default(); - let shadow_autonomous_clock_valid_nodes = shadow_autonomous_clock_valid_scrapers.len(); - let ( - shadow_autonomous_clock_carrier_round_min, - shadow_autonomous_clock_carrier_round_max, - shadow_autonomous_clock_phase_backlog_total, - shadow_autonomous_clock_admitted_authors_min, - shadow_autonomous_clock_admitted_stake_min, - shadow_autonomous_clock_buffered_authenticated_total, - ) = if shadow_autonomous_clock_enabled { - ( - self.min_latest_scalar_as_usize("starfish_rbc_dag_shadow_carrier_round"), - self.max_result("starfish_rbc_dag_shadow_carrier_round", |measurement| { - measurement.scalar.max(0.0) as usize - }), - self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_phase_backlog"), - self.min_latest_scalar_as_usize("starfish_rbc_dag_shadow_admitted_authors"), - self.min_latest_scalar_as_usize("starfish_rbc_dag_shadow_admitted_stake"), - self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_buffered_authenticated"), - ) - } else { - (0, 0, 0, 0, 0, 0) - }; - let autonomous_phase_backlog_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR - .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let autonomous_buffered_bound = STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR - .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let every_autonomous_scraper_has_progress = - shadow_autonomous_clock_valid_scrapers - .iter() - .all(|scraper_id| { - self.count_bucket_increased( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - "heartbeat,accepted", - ) && self.count_bucket_increased( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - "delivery,shadow", - ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_durable_batches_total", - *scraper_id, - ) && self.scalar_counter_increased( - "starfish_rbc_dag_shadow_wal_durable_records_total", - *scraper_id, - ) && self.scalar_gauge_increased( - "starfish_rbc_dag_shadow_carrier_round", - *scraper_id, - ) && self.latest_scalar_greater_than( - "starfish_rbc_dag_shadow_carrier_round", - *scraper_id, - 1.0, - ) && self.latest_scalar_equals( - "starfish_rbc_dag_shadow_pending_recovery", - *scraper_id, - 0.0, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_phase_backlog", - *scraper_id, - autonomous_phase_backlog_bound as f64, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_admitted_authors", - *scraper_id, - self.parameters.nodes as f64, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_admitted_stake", - *scraper_id, - f64::MAX, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_buffered_authenticated", - *scraper_id, - autonomous_buffered_bound as f64, - ) - }); - let shadow_autonomous_clock_valid = shadow_autonomous_clock_enabled - && expected_shadow_nodes != 0 - && shadow_autonomous_clock_valid_nodes == expected_shadow_nodes - && every_autonomous_scraper_has_progress - && shadow_autonomous_clock_carrier_round_max - .saturating_sub(shadow_autonomous_clock_carrier_round_min) - <= usize::try_from(STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG).unwrap_or(usize::MAX); BenchmarkRunSummary { protocol: self.parameters.consensus_protocol.clone(), @@ -1237,15 +1021,6 @@ impl MeasurementsCollection { shadow_unpaired_direct, shadow_unpaired_shadow, shadow_unpaired_max_round_lag, - shadow_autonomous_clock_enabled, - shadow_autonomous_clock_valid, - shadow_autonomous_clock_valid_nodes, - shadow_autonomous_clock_carrier_round_min, - shadow_autonomous_clock_carrier_round_max, - shadow_autonomous_clock_phase_backlog_total, - shadow_autonomous_clock_admitted_authors_min, - shadow_autonomous_clock_admitted_stake_min, - shadow_autonomous_clock_buffered_authenticated_total, } } @@ -1318,28 +1093,6 @@ impl MeasurementsCollection { ) ]); } - if summary.shadow_autonomous_clock_enabled { - table.add_row(row![ - b->"RBC-DAG autonomous clock:", - format!( - "valid={} ({}/{} validators), carrier rounds={}..={}, embedded deliveries={}, phase backlog={}, \ - admitted authors/stake min={}/{}, buffered authenticated={}, WAL records={}, \ - pending recovery={}", - summary.shadow_autonomous_clock_valid, - summary.shadow_autonomous_clock_valid_nodes, - summary.ready_nodes_at_boot, - summary.shadow_autonomous_clock_carrier_round_min, - summary.shadow_autonomous_clock_carrier_round_max, - summary.shadow_deliveries, - summary.shadow_autonomous_clock_phase_backlog_total, - summary.shadow_autonomous_clock_admitted_authors_min, - summary.shadow_autonomous_clock_admitted_stake_min, - summary.shadow_autonomous_clock_buffered_authenticated_total, - summary.shadow_wal_durable_records, - summary.shadow_pending_recovery, - ) - ]); - } table.add_row(row![ b->"End-to-end latency:", format!( @@ -1424,12 +1177,6 @@ mod test { parameters } - fn autonomous_shadow_benchmark_parameters(nodes: usize) -> BenchmarkParameters { - let mut parameters = shadow_benchmark_parameters(nodes); - parameters.node_parameters.starfish_rbc_dag_autonomous_clock = true; - parameters - } - #[allow(clippy::too_many_arguments)] fn add_shadow_snapshot( collection: &mut MeasurementsCollection, @@ -1442,7 +1189,7 @@ mod test { direct_only: usize, shadow_only: usize, ambiguous: usize, - wal_durable_records: usize, + wal_durable_records: f64, ) { collection.add( scraper_id, @@ -1483,8 +1230,8 @@ mod test { scraper_id, "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), Measurement { - count: wal_durable_records, - scalar: wal_durable_records as f64, + count: wal_durable_records as usize, + scalar: wal_durable_records, ..Measurement::default() }, ); @@ -1522,95 +1269,6 @@ mod test { } } - #[allow(clippy::too_many_arguments)] - fn add_autonomous_clock_snapshot( - collection: &mut MeasurementsCollection, - scraper_id: usize, - clock_valid: f64, - carrier_round: usize, - phase_backlog: usize, - admitted_authors: usize, - admitted_stake: usize, - buffered_authenticated: usize, - wal_durable_records: usize, - ) { - let timestamp = Duration::from_secs(carrier_round as u64); - for (label, value) in [ - ("starfish_rbc_dag_shadow_clock_valid", clock_valid), - ( - "starfish_rbc_dag_shadow_carrier_round", - carrier_round as f64, - ), - ( - "starfish_rbc_dag_shadow_phase_backlog", - phase_backlog as f64, - ), - ( - "starfish_rbc_dag_shadow_admitted_authors", - admitted_authors as f64, - ), - ( - "starfish_rbc_dag_shadow_admitted_stake", - admitted_stake as f64, - ), - ( - "starfish_rbc_dag_shadow_buffered_authenticated", - buffered_authenticated as f64, - ), - ] { - collection.add( - scraper_id, - label.to_owned(), - Measurement { - timestamp, - scalar: value, - ..Measurement::default() - }, - ); - } - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_inputs_total".to_owned(), - Measurement { - timestamp, - count_buckets: HashMap::from([ - ("heartbeat,accepted".to_owned(), wal_durable_records), - ("delivery,shadow".to_owned(), wal_durable_records), - ]), - count: wal_durable_records.saturating_mul(2), - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_wal_durable_batches_total".to_owned(), - Measurement { - timestamp, - count: wal_durable_records, - scalar: wal_durable_records as f64, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), - Measurement { - timestamp, - count: wal_durable_records, - scalar: wal_durable_records as f64, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_pending_recovery".to_owned(), - Measurement { - timestamp, - ..Measurement::default() - }, - ); - } - #[test] fn average_latency() { let data = Measurement { @@ -1831,98 +1489,6 @@ starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 assert_eq!(summary.shadow_unpaired_max_round_lag, 1); } - #[test] - fn prometheus_parse_preserves_autonomous_clock_state() { - let report = r#" -# TYPE benchmark_duration counter -benchmark_duration 30 -# TYPE starfish_rbc_dag_shadow_clock_valid gauge -starfish_rbc_dag_shadow_clock_valid 1 -# TYPE starfish_rbc_dag_shadow_carrier_round gauge -starfish_rbc_dag_shadow_carrier_round 12 -# TYPE starfish_rbc_dag_shadow_phase_backlog gauge -starfish_rbc_dag_shadow_phase_backlog 3 -# TYPE starfish_rbc_dag_shadow_admitted_authors gauge -starfish_rbc_dag_shadow_admitted_authors 4 -# TYPE starfish_rbc_dag_shadow_admitted_stake gauge -starfish_rbc_dag_shadow_admitted_stake 7 -# TYPE starfish_rbc_dag_shadow_buffered_authenticated gauge -starfish_rbc_dag_shadow_buffered_authenticated 2 -"#; - - let measurements = Measurement::from_prometheus::(report); - for (label, expected) in [ - ("starfish_rbc_dag_shadow_clock_valid", 1.0), - ("starfish_rbc_dag_shadow_carrier_round", 12.0), - ("starfish_rbc_dag_shadow_phase_backlog", 3.0), - ("starfish_rbc_dag_shadow_admitted_authors", 4.0), - ("starfish_rbc_dag_shadow_admitted_stake", 7.0), - ("starfish_rbc_dag_shadow_buffered_authenticated", 2.0), - ] { - assert_eq!(measurements[label].scalar, expected, "metric {label}"); - } - } - - #[test] - fn autonomous_clock_has_a_distinct_sticky_summary() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(2)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 1, 2, 7, 1, 5); - add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 8, 1, 2, 6, 1, 6); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 3, 2, 7, 2, 10); - add_autonomous_clock_snapshot(&mut collection, 1, 1.0, 10, 4, 2, 6, 3, 11); - - let summary = collection.benchmark_run_summary(); - assert!(!summary.shadow_comparison_enabled); - assert!(!summary.shadow_comparison_valid); - assert!(summary.shadow_autonomous_clock_enabled); - assert!(summary.shadow_autonomous_clock_valid); - assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 2); - assert_eq!(summary.shadow_autonomous_clock_carrier_round_min, 10); - assert_eq!(summary.shadow_autonomous_clock_carrier_round_max, 12); - assert_eq!(summary.shadow_autonomous_clock_phase_backlog_total, 7); - assert_eq!(summary.shadow_autonomous_clock_admitted_authors_min, 2); - assert_eq!(summary.shadow_autonomous_clock_admitted_stake_min, 6); - assert_eq!( - summary.shadow_autonomous_clock_buffered_authenticated_total, - 5 - ); - assert_eq!(summary.shadow_wal_durable_records, 21); - - // A later healthy scrape must not erase an earlier invalid verdict. - add_autonomous_clock_snapshot(&mut collection, 0, 0.0, 13, 0, 3, 7, 0, 12); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 14, 0, 3, 7, 0, 13); - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 1); - assert!(!summary.shadow_autonomous_clock_valid); - } - - #[test] - fn missing_final_autonomous_scrape_invalidates_only_clock_verdict() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 7, 0, 5); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 0, 1, 7, 0, 10); - assert!( - collection - .benchmark_run_summary() - .shadow_autonomous_clock_valid - ); - - collection.mark_shadow_final_scrape_missing(0); - - let validity = collection - .scraper_series("starfish_rbc_dag_shadow_clock_valid", 0) - .unwrap(); - assert_eq!(validity.last().unwrap().scalar_value(), 0.0); - assert!( - collection - .scraper_series("starfish_rbc_dag_shadow_comparison_valid", 0) - .is_none() - ); - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 0); - assert!(!summary.shadow_autonomous_clock_valid); - } - #[test] fn shadow_verdict_remains_invalid_after_historical_failure_and_counter_reset() { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); @@ -1931,8 +1497,8 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 // comparison category. The second scrape deliberately looks clean, // including reset comparison counters, so a latest-value-only verdict // would incorrectly accept the run. - add_shadow_snapshot(&mut collection, 0, 0.0, 1, 1, 1, 1, 1, 1, 1, 1); - add_shadow_snapshot(&mut collection, 0, 1.0, 2, 2, 2, 0, 0, 0, 0, 2); + add_shadow_snapshot(&mut collection, 0, 0.0, 1, 1, 1, 1, 1, 1, 1, 1.0); + add_shadow_snapshot(&mut collection, 0, 1.0, 2, 2, 2, 0, 0, 0, 0, 2.0); assert!(!collection.gauge_always_equals( "starfish_rbc_dag_shadow_comparison_valid", @@ -1953,100 +1519,10 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 assert!(!summary.shadow_comparison_valid); } - #[test] - fn autonomous_clock_valid_gauge_without_durable_progress_is_invalid() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); - - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_autonomous_clock_valid_nodes, 1); - assert!(!summary.shadow_autonomous_clock_valid); - } - - #[test] - fn autonomous_clock_warmup_only_progress_is_invalid() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 2, 0, 1, 1, 0, 3); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 4, 0, 1, 1, 0, 6); - for by_scraper in collection.data.values_mut() { - for measurement in by_scraper.get_mut(&0).into_iter().flatten() { - measurement.timestamp = Duration::ZERO; - } - } - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 4, 0, 1, 1, 0, 6); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 4, 0, 1, 1, 0, 6); - for by_scraper in collection.data.values_mut() { - if let Some(last) = by_scraper.get_mut(&0).and_then(|series| series.last_mut()) { - last.timestamp = Duration::from_secs(5); - } - } - - assert!( - !collection - .benchmark_run_summary() - .shadow_autonomous_clock_valid - ); - } - - #[test] - fn autonomous_clock_verdict_requires_in_window_rbc_delivery() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 10, 0, 1, 1, 0, 6); - collection - .data - .get_mut("starfish_rbc_dag_shadow_inputs_total") - .and_then(|by_scraper| by_scraper.get_mut(&0)) - .and_then(|series| series.last_mut()) - .unwrap() - .count_buckets - .insert("delivery,shadow".to_owned(), 3); - - assert!( - !collection - .benchmark_run_summary() - .shadow_autonomous_clock_valid - ); - } - - #[test] - fn autonomous_clock_verdict_rejects_observed_round_rollback() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 1, 0, 1, 1, 0, 4); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 9, 0, 1, 1, 0, 5); - - assert!( - !collection - .benchmark_run_summary() - .shadow_autonomous_clock_valid - ); - } - - #[test] - fn autonomous_clock_verdict_rejects_benchmark_timestamp_reset() { - let mut collection = MeasurementsCollection::new(autonomous_shadow_benchmark_parameters(1)); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 8, 0, 1, 1, 0, 3); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 10, 0, 1, 1, 0, 5); - add_autonomous_clock_snapshot(&mut collection, 0, 1.0, 12, 0, 1, 1, 0, 7); - for by_scraper in collection.data.values_mut() { - if let Some(last) = by_scraper.get_mut(&0).and_then(|series| series.last_mut()) { - last.timestamp = Duration::from_secs(1); - } - } - - assert!( - !collection - .benchmark_run_summary() - .shadow_autonomous_clock_valid - ); - } - #[test] fn missing_final_shadow_scrape_invalidates_a_previously_valid_snapshot() { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8); + add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); assert!(collection.benchmark_run_summary().shadow_comparison_valid); collection.mark_shadow_final_scrape_missing(0); @@ -2073,8 +1549,8 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 #[test] fn shadow_verdict_requires_delivery_and_wal_coverage_from_every_scraper() { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(2)); - add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8); - add_shadow_snapshot(&mut collection, 1, 1.0, 0, 0, 0, 0, 0, 0, 0, 0); + add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); + add_shadow_snapshot(&mut collection, 1, 1.0, 0, 0, 0, 0, 0, 0, 0, 0.0); let summary = collection.benchmark_run_summary(); assert_eq!(summary.shadow_comparison_valid_nodes, 2); @@ -2100,7 +1576,7 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 0, 0, 0, - 8, + 8.0, ); add_shadow_backlog_snapshot( &mut collection, @@ -2122,7 +1598,7 @@ starfish_rbc_dag_shadow_buffered_authenticated 2 fn shadow_verdict_rejects_excessive_or_old_unpaired_work() { for (unpaired_direct, unpaired_shadow, max_round_lag) in [(5, 0, 1), (0, 5, 1), (1, 0, 5)] { let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot(&mut collection, 0, 1.0, 8, 7, 7, 0, 0, 0, 0, 8); + add_shadow_snapshot(&mut collection, 0, 1.0, 8, 7, 7, 0, 0, 0, 0, 8.0); add_shadow_backlog_snapshot( &mut collection, 0, diff --git a/crates/orchestrator/src/orchestrator.rs b/crates/orchestrator/src/orchestrator.rs index 70871516..6ce99bd4 100644 --- a/crates/orchestrator/src/orchestrator.rs +++ b/crates/orchestrator/src/orchestrator.rs @@ -25,7 +25,7 @@ use crate::{ error::{SshError, TestbedError, TestbedResult}, faults::{CrashRecoverySchedule, FaultsType}, logs::LogsAnalyzer, - measurements::{Measurement, MeasurementsCollection, shadow_validity_metric}, + measurements::{Measurement, MeasurementsCollection}, monitor::Monitor, protocol::{ProtocolCommands, ProtocolMetrics}, settings::Settings, @@ -1098,8 +1098,8 @@ impl Orchestrator

{ .filter(|node| !killed_node_ids.contains(&node.id)) .filter_map(|node| node_indices.get(&node.id).copied()) .collect(); - let shadow_final_validity_metric = shadow_validity_metric(parameters); - let shadow_final_scrape_required = shadow_final_validity_metric.is_some(); + let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" + && parameters.node_parameters.starfish_rbc_dag_shadow; let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1224,9 +1224,7 @@ impl Orchestrator

{ continue; }; let parsed = Measurement::from_prometheus::

(stdout); - if shadow_final_validity_metric - .is_some_and(|metric| parsed.contains_key(metric)) - { + if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { fresh_final_shadow_scrapers.insert(i); } for (label, measurement) in parsed { @@ -1432,8 +1430,8 @@ impl Orchestrator

{ .filter(|node| !killed_node_ids.contains(&node.id)) .filter_map(|node| node_indices.get(&node.id).copied()) .collect(); - let shadow_final_validity_metric = shadow_validity_metric(parameters); - let shadow_final_scrape_required = shadow_final_validity_metric.is_some(); + let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" + && parameters.node_parameters.starfish_rbc_dag_shadow; let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1847,7 +1845,7 @@ impl Orchestrator

{ continue; }; let parsed = Measurement::from_prometheus::

(stdout); - if shadow_final_validity_metric.is_some_and(|metric| parsed.contains_key(metric)) { + if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { fresh_final_shadow_scrapers.insert(i); } for (label, measurement) in parsed { diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 8b3bc56c..e5bcdf1f 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -19,10 +19,7 @@ use super::{ BINARY_PATH, METRICS_CURL_CONNECT_TIMEOUT_SECS, METRICS_CURL_MAX_TIME_SECS, ProtocolCommands, ProtocolMetrics, ProtocolParameters, }; -use crate::{ - benchmark::BenchmarkParameters, client::Instance, measurements::shadow_validity_metric, - settings::Settings, -}; +use crate::{benchmark::BenchmarkParameters, client::Instance, settings::Settings}; #[derive(Clone, Serialize, Deserialize, Default)] #[serde(transparent)] @@ -278,8 +275,6 @@ impl ProtocolMetrics for StarfishProtocol { .collect(); } - let validity_metric = shadow_validity_metric(parameters) - .expect("Starfish-RBC shadow readiness has an active validity metric"); self.nodes_metrics_path(instances, parameters) .into_iter() .map(|(instance, path)| { @@ -289,7 +284,7 @@ impl ProtocolMetrics for StarfishProtocol { "curl --silent --show-error --fail --compressed --connect-timeout \ {METRICS_CURL_CONNECT_TIMEOUT_SECS} --max-time \ {METRICS_CURL_MAX_TIME_SECS} {path} | grep -Eq \ - '^{validity_metric}(\\{{[^}}]*\\}})? \ + '^starfish_rbc_dag_shadow_comparison_valid(\\{{[^}}]*\\}})? \ 1(\\.0)?$'" ), ) @@ -323,9 +318,6 @@ impl StarfishProtocol { // not let it make Sailfish++ or another comparison member fail // validator configuration. node_parameters.starfish_rbc_dag_shadow = false; - node_parameters.starfish_rbc_dag_autonomous_clock = false; - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = - config::node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(); } node_parameters } @@ -362,14 +354,12 @@ impl StarfishProtocol { mod tests { use super::{ProtocolMetrics, StarfishNodeParameters, StarfishProtocol}; use crate::{benchmark::BenchmarkParameters, client::Instance}; - use starfish_core::config::{NodeParameters, node_defaults}; + use starfish_core::config::NodeParameters; #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, - starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }); let parameters = @@ -380,16 +370,12 @@ mod tests { .is_some_and(|instance| instance != [0; 32]) ); assert!(parameters.starfish_rbc_dag_shadow); - assert!(parameters.starfish_rbc_dag_autonomous_clock); - assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } #[test] fn non_rbc_genesis_does_not_need_a_protocol_instance() { let shared_parameters = StarfishNodeParameters(NodeParameters { starfish_rbc_dag_shadow: true, - starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }); let parameters = @@ -399,15 +385,6 @@ mod tests { !parameters.starfish_rbc_dag_shadow, "a global shadow flag must not leak into non-RBC comparison members" ); - assert!( - !parameters.starfish_rbc_dag_autonomous_clock, - "a global autonomous-clock flag must not leak into non-RBC comparison members" - ); - assert_eq!( - parameters.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), - "a global heartbeat override must not leak into non-RBC comparison members" - ); } #[test] @@ -428,26 +405,6 @@ mod tests { assert!(command.contains("grep -Eq")); } - #[test] - fn autonomous_shadow_readiness_waits_for_clock_verdict() { - let protocol = StarfishProtocol { - working_dir: std::path::PathBuf::from("benchmark"), - }; - let mut parameters = BenchmarkParameters::new_for_tests(); - parameters.consensus_protocol = "starfish-rbc".to_owned(); - parameters.node_parameters.starfish_rbc_dag_shadow = true; - parameters.node_parameters.starfish_rbc_dag_autonomous_clock = true; - let command = protocol - .nodes_readiness_command(vec![Instance::new_for_test("1".into())], ¶meters) - .pop() - .unwrap() - .1; - - assert!(command.contains("starfish_rbc_dag_shadow_clock_valid")); - assert!(!command.contains("starfish_rbc_dag_shadow_comparison_valid")); - assert!(command.contains("grep -Eq")); - } - #[test] fn split_authority_load_preserves_total_load() { let nodes = 4; diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 6a455bb4..5910be63 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -72,11 +72,6 @@ pub struct NodeParameters { /// observational only and cannot affect the DAG, pacemaker, or commits. #[serde(default)] pub starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create an autonomous - /// optimistic carrier clock. This remains experimental and requires - /// `starfish_rbc_dag_shadow`. - #[serde(default)] - pub starfish_rbc_dag_autonomous_clock: bool, /// Testbed-only receiver-local single-DAG RBC path: deliver an exact header /// after locally observing quorum ECHO rather than quorum READY. Quorum /// intersection preserves a unique value, but pairwise-MAC testimony is @@ -84,10 +79,6 @@ pub struct NodeParameters { /// totality. This must remain an explicit benchmark flag. #[serde(default)] pub starfish_rbc_single_dag_echo_qc_fast_path: bool, - /// Maximum interval between autonomous Starfish-RBC-DAG heartbeat - /// carriers. The value is ignored unless the autonomous clock is enabled. - #[serde(default = "node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms")] - pub starfish_rbc_dag_heartbeat_interval_ms: u64, #[serde(default = "node_defaults::default_causal_push_shard_round_lag")] pub causal_push_shard_round_lag: RoundNumber, #[serde( @@ -138,10 +129,6 @@ pub mod node_defaults { 5 } - pub fn default_starfish_rbc_dag_heartbeat_interval_ms() -> u64 { - 250 - } - pub fn default_causal_push_shard_round_lag() -> RoundNumber { 0 } @@ -168,10 +155,7 @@ impl Default for NodeParameters { block_authentication: None, starfish_rbc_protocol_instance: None, starfish_rbc_dag_shadow: false, - starfish_rbc_dag_autonomous_clock: false, starfish_rbc_single_dag_echo_qc_fast_path: false, - starfish_rbc_dag_heartbeat_interval_ms: - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms(), causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: node_defaults::default_enable_strong_vote_adaptive_acknowledgments(), @@ -409,36 +393,20 @@ impl NodePrivateConfig { pub fn starfish_rbc_dag_shadow_wal(&self) -> PathBuf { self.storage_path.join("starfish-rbc-dag-shadow-v1.wal") } - - pub fn starfish_rbc_dag_autonomous_clock_wal(&self) -> PathBuf { - self.storage_path - .join("starfish-rbc-dag-autonomous-clock-v1.wal") - } } impl ImportExport for NodePrivateConfig {} #[cfg(test)] mod tests { - use std::path::Path; - - use super::{NodeParameters, NodePrivateConfig, node_defaults}; + use super::NodeParameters; #[test] fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { let mut parameters: NodeParameters = serde_yaml::from_str("{}").unwrap(); assert_eq!(parameters.starfish_rbc_protocol_instance, None); assert!(!parameters.starfish_rbc_dag_shadow); - assert!(!parameters.starfish_rbc_dag_autonomous_clock); assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); - assert_eq!( - parameters.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() - ); - assert_eq!( - parameters.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() - ); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); assert_ne!(protocol_instance, [0; 32]); @@ -450,49 +418,7 @@ mod tests { Some(protocol_instance) ); assert!(!decoded.starfish_rbc_dag_shadow); - assert!(!decoded.starfish_rbc_dag_autonomous_clock); assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); - assert_eq!( - decoded.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() - ); - assert_eq!( - decoded.starfish_rbc_dag_heartbeat_interval_ms, - node_defaults::default_starfish_rbc_dag_heartbeat_interval_ms() - ); - } - - #[test] - fn autonomous_clock_configuration_roundtrips() { - let parameters = NodeParameters { - starfish_rbc_dag_shadow: true, - starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, - ..NodeParameters::default() - }; - - let yaml = serde_yaml::to_string(¶meters).unwrap(); - let decoded: NodeParameters = serde_yaml::from_str(&yaml).unwrap(); - assert!(decoded.starfish_rbc_dag_shadow); - assert!(decoded.starfish_rbc_dag_autonomous_clock); - assert_eq!(decoded.starfish_rbc_dag_heartbeat_interval_ms, 125); - } - - #[test] - fn autonomous_clock_uses_a_distinct_wal() { - let private_config = - NodePrivateConfig::new_for_benchmarks(Path::new("benchmark"), 1).remove(0); - - assert_ne!( - private_config.starfish_rbc_dag_shadow_wal(), - private_config.starfish_rbc_dag_autonomous_clock_wal() - ); - assert_eq!( - private_config.starfish_rbc_dag_autonomous_clock_wal(), - Path::new("benchmark") - .join("storage-0") - .join("starfish-rbc-dag-autonomous-clock-v1.wal") - ); } } diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index cbb56c7a..83487543 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -41,16 +41,6 @@ pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR: i64 = 4; pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; -/// Local-benchmark guards for the non-authoritative autonomous carrier -/// clock. The round-skew limit matches the executable model's bounded future -/// buffer. A healthy clock can transiently retain phase work, but its carrier -/// capacity exceeds the two RBC statements generated per admitted value; a -/// sixteen-committee backlog therefore leaves generous scheduling headroom -/// while still detecting an actor that is no longer draining work. -pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG: i64 = 4; -pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR: i64 = 16; -pub const STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR: i64 = 2; - #[derive(Clone)] pub struct Metrics { pub benchmark_duration: IntCounter, @@ -164,12 +154,6 @@ pub struct Metrics { pub starfish_rbc_dag_shadow_unpaired_shadow: IntGauge, pub starfish_rbc_dag_shadow_unpaired_max_round_lag: IntGauge, pub starfish_rbc_dag_shadow_comparison_valid: IntGauge, - pub starfish_rbc_dag_shadow_clock_valid: IntGauge, - pub starfish_rbc_dag_shadow_carrier_round: IntGauge, - pub starfish_rbc_dag_shadow_phase_backlog: IntGauge, - pub starfish_rbc_dag_shadow_admitted_authors: IntGauge, - pub starfish_rbc_dag_shadow_admitted_stake: IntGauge, - pub starfish_rbc_dag_shadow_buffered_authenticated: IntGauge, // subscription tracking pub subscribed_to_peers: IntGauge, @@ -233,199 +217,6 @@ pub struct MetricReporter { pub global_in_memory_blocks_bytes: IntGauge, } -/// Per-validator counters captured after the autonomous shadow becomes ready -/// and immediately before the measured local-benchmark interval begins. -#[derive(Clone, Copy, Debug, Default)] -pub struct AutonomousClockBenchmarkBaseline { - accepted_heartbeats: u64, - delivered_carriers: u64, - wal_batches: u64, - wal_records: u64, - carrier_round: i64, -} - -#[derive(Debug, Eq, PartialEq)] -struct AutonomousClockBenchmarkSummary { - valid_nodes: usize, - progress_nodes: usize, - bounded_nodes: usize, - accepted_heartbeats: u64, - delivered_carriers: u64, - wal_batches: u64, - wal_records: u64, - pending_recovery: i64, - minimum_round: i64, - maximum_round: i64, - maximum_phase_backlog: i64, - maximum_admitted_authors: i64, - maximum_admitted_stake: i64, - maximum_buffered_authenticated: i64, - maximum_phase_backlog_bound: i64, - maximum_buffered_authenticated_bound: i64, - verdict_valid: bool, -} - -fn summarize_autonomous_clock_benchmark( - metrics: &[Arc], - committee_size: usize, - baselines: Option<&[AutonomousClockBenchmarkBaseline]>, -) -> AutonomousClockBenchmarkSummary { - let committee_size = i64::try_from(committee_size).unwrap_or(i64::MAX); - let maximum_phase_backlog_bound = - STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR.saturating_mul(committee_size); - let maximum_buffered_authenticated_bound = - STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR.saturating_mul(committee_size); - - let valid_nodes = metrics - .iter() - .filter(|metrics| metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1) - .count(); - let progress_nodes = metrics - .iter() - .enumerate() - .filter(|(index, metrics)| { - let baseline = baselines - .and_then(|baselines| baselines.get(*index)) - .copied() - .unwrap_or_default(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .get() - > baseline.accepted_heartbeats - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get() - > baseline.delivered_carriers - && metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .get() - > baseline.wal_batches - && metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .get() - > baseline.wal_records - && metrics.starfish_rbc_dag_shadow_carrier_round.get() > baseline.carrier_round - }) - .count(); - let bounded_nodes = metrics - .iter() - .filter(|metrics| { - let phase_backlog = metrics.starfish_rbc_dag_shadow_phase_backlog.get(); - let admitted_authors = metrics.starfish_rbc_dag_shadow_admitted_authors.get(); - let admitted_stake = metrics.starfish_rbc_dag_shadow_admitted_stake.get(); - let buffered = metrics.starfish_rbc_dag_shadow_buffered_authenticated.get(); - phase_backlog >= 0 - && phase_backlog <= maximum_phase_backlog_bound - && admitted_authors >= 0 - && admitted_authors <= committee_size - && admitted_stake >= 0 - && buffered >= 0 - && buffered <= maximum_buffered_authenticated_bound - && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 - }) - .count(); - - let accepted_heartbeats = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .get() - }) - .sum(); - let delivered_carriers = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get() - }) - .sum(); - let wal_batches = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .get() - }) - .sum(); - let wal_records = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .get() - }) - .sum(); - let pending_recovery = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_pending_recovery.get()) - .sum(); - let minimum_round = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) - .min() - .unwrap_or_default(); - let maximum_round = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_carrier_round.get()) - .max() - .unwrap_or_default(); - let maximum_phase_backlog = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_phase_backlog.get()) - .max() - .unwrap_or_default(); - let maximum_admitted_authors = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_authors.get()) - .max() - .unwrap_or_default(); - let maximum_admitted_stake = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_admitted_stake.get()) - .max() - .unwrap_or_default(); - let maximum_buffered_authenticated = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_buffered_authenticated.get()) - .max() - .unwrap_or_default(); - let round_lag = maximum_round.saturating_sub(minimum_round); - let every_node_valid = valid_nodes == metrics.len(); - let every_node_progressed = progress_nodes == metrics.len(); - let every_node_bounded = bounded_nodes == metrics.len(); - let verdict_valid = !metrics.is_empty() - && every_node_valid - && every_node_progressed - && every_node_bounded - && round_lag <= STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG; - - AutonomousClockBenchmarkSummary { - valid_nodes, - progress_nodes, - bounded_nodes, - accepted_heartbeats, - delivered_carriers, - wal_batches, - wal_records, - pending_recovery, - minimum_round, - maximum_round, - maximum_phase_backlog, - maximum_admitted_authors, - maximum_admitted_stake, - maximum_buffered_authenticated, - maximum_phase_backlog_bound, - maximum_buffered_authenticated_bound, - verdict_valid, - } -} - pub struct HistogramReporter { pub histogram: PreciseHistogram, gauge: IntGaugeVec, @@ -437,22 +228,6 @@ pub struct VecHistogramReporter { } impl Metrics { - pub fn autonomous_clock_benchmark_baseline(&self) -> AutonomousClockBenchmarkBaseline { - AutonomousClockBenchmarkBaseline { - accepted_heartbeats: self - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .get(), - delivered_carriers: self - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get(), - wal_batches: self.starfish_rbc_dag_shadow_wal_durable_batches_total.get(), - wal_records: self.starfish_rbc_dag_shadow_wal_durable_records_total.get(), - carrier_round: self.starfish_rbc_dag_shadow_carrier_round.get(), - } - } - pub fn new( registry: &Registry, committee: Option<&Committee>, @@ -831,42 +606,6 @@ impl Metrics { registry, ) .unwrap(), - starfish_rbc_dag_shadow_clock_valid: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_clock_valid", - "State of the non-authoritative autonomous carrier clock (1 valid, 0 disabled/invalid, -1 starting)", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_carrier_round: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_carrier_round", - "Currently open sequential Starfish-RBC-DAG shadow carrier round", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_phase_backlog: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_phase_backlog", - "Pending embedded ECHO/READY statements in the autonomous carrier actor", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_admitted_authors: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_admitted_authors", - "Distinct authors admitted in the currently open carrier round", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_admitted_stake: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_admitted_stake", - "Stake admitted in the currently open carrier round", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_buffered_authenticated: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_buffered_authenticated", - "Authenticated future carrier slots buffered outside the current admission window", - registry, - ) - .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1325,8 +1064,6 @@ impl Metrics { duration_secs: u64, committee_size: usize, starfish_rbc_dag_shadow_expected: bool, - starfish_rbc_dag_autonomous_clock_expected: bool, - autonomous_clock_baselines: Option>, ) { let num_validators = metrics.len() as u64; @@ -1467,8 +1204,6 @@ impl Metrics { "rbc_dag_shadow_carrier", "rbc_dag_shadow_carrier_request", "rbc_dag_shadow_carrier_response", - "rbc_dag_shadow_carrier_sync_request", - "rbc_dag_shadow_carrier_sync_response", ]; let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() @@ -1524,64 +1259,7 @@ impl Metrics { }; table.add_row(row![b->"Bandwidth efficiency:", format!("{:.2}", bandwidth_efficiency)]); - if starfish_rbc_dag_autonomous_clock_expected { - let summary = summarize_autonomous_clock_benchmark( - &metrics, - committee_size, - autonomous_clock_baselines.as_deref(), - ); - let round_lag = summary.maximum_round.saturating_sub(summary.minimum_round); - - table.add_row(row![bH2->""]); - table.add_row(row![bH2->"RBC-DAG Autonomous Clock Verification"]); - table.add_row(row![ - b->"Clock verdict:", - if summary.verdict_valid { - "VALID".to_owned() - } else { - "INVALID — DISCARD THIS AUTONOMOUS-CLOCK RUN".to_owned() - } - ]); - table.add_row(row![ - b->"Valid/progress/bounded validators:", - format!( - "{}/{}, {}/{}, {}/{}", - summary.valid_nodes, - metrics.len(), - summary.progress_nodes, - metrics.len(), - summary.bounded_nodes, - metrics.len(), - ) - ]); - table.add_row(row![ - b->"Durable clock progress:", - format!( - "heartbeats={}, RBC deliveries={}, WAL batches={}, records={}, open rounds={}..{}", - summary.accepted_heartbeats, - summary.delivered_carriers, - summary.wal_batches, - summary.wal_records, - summary.minimum_round, - summary.maximum_round, - ) - ]); - table.add_row(row![ - b->"Bounded live state:", - format!( - "round skew={round_lag}/{}, max phase backlog={}/{}, admitted authors={}/{}, stake={}, max buffered={}/{}, pending recovery={}", - STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG, - summary.maximum_phase_backlog, - summary.maximum_phase_backlog_bound, - summary.maximum_admitted_authors, - committee_size, - summary.maximum_admitted_stake, - summary.maximum_buffered_authenticated, - summary.maximum_buffered_authenticated_bound, - summary.pending_recovery, - ) - ]); - } else if starfish_rbc_dag_shadow_expected { + if starfish_rbc_dag_shadow_expected { let valid_nodes = metrics .iter() .filter(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) @@ -2062,14 +1740,6 @@ mod tests { .starfish_rbc_dag_shadow_unpaired_max_round_lag .set(8); metrics.starfish_rbc_dag_shadow_comparison_valid.set(1); - metrics.starfish_rbc_dag_shadow_clock_valid.set(1); - metrics.starfish_rbc_dag_shadow_carrier_round.set(9); - metrics.starfish_rbc_dag_shadow_phase_backlog.set(10); - metrics.starfish_rbc_dag_shadow_admitted_authors.set(3); - metrics.starfish_rbc_dag_shadow_admitted_stake.set(3); - metrics - .starfish_rbc_dag_shadow_buffered_authenticated - .set(2); let gathered = registry.gather(); for name in [ @@ -2084,12 +1754,6 @@ mod tests { "starfish_rbc_dag_shadow_unpaired_shadow", "starfish_rbc_dag_shadow_unpaired_max_round_lag", "starfish_rbc_dag_shadow_comparison_valid", - "starfish_rbc_dag_shadow_clock_valid", - "starfish_rbc_dag_shadow_carrier_round", - "starfish_rbc_dag_shadow_phase_backlog", - "starfish_rbc_dag_shadow_admitted_authors", - "starfish_rbc_dag_shadow_admitted_stake", - "starfish_rbc_dag_shadow_buffered_authenticated", ] { assert!( gathered.iter().any(|family| family.get_name() == name), @@ -2097,123 +1761,4 @@ mod tests { ); } } - - fn autonomous_clock_metrics( - round: i64, - phase_backlog: i64, - buffered_authenticated: i64, - ) -> Arc { - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new(®istry, None, None, None); - metrics.starfish_rbc_dag_shadow_clock_valid.set(1); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .inc_by(2); - metrics.starfish_rbc_dag_shadow_carrier_round.set(round); - metrics - .starfish_rbc_dag_shadow_phase_backlog - .set(phase_backlog); - metrics.starfish_rbc_dag_shadow_admitted_authors.set(2); - metrics.starfish_rbc_dag_shadow_admitted_stake.set(2); - metrics - .starfish_rbc_dag_shadow_buffered_authenticated - .set(buffered_authenticated); - metrics - } - - #[test] - fn autonomous_clock_summary_requires_every_node_to_make_bounded_durable_progress() { - let metrics = vec![ - autonomous_clock_metrics(8, 3, 1), - autonomous_clock_metrics(9, 4, 2), - autonomous_clock_metrics(10, 5, 0), - autonomous_clock_metrics(11, 6, 1), - ]; - - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); - - assert!(summary.verdict_valid); - assert_eq!(summary.valid_nodes, 4); - assert_eq!(summary.progress_nodes, 4); - assert_eq!(summary.bounded_nodes, 4); - assert_eq!(summary.accepted_heartbeats, 4); - assert_eq!(summary.delivered_carriers, 4); - assert_eq!(summary.wal_batches, 4); - assert_eq!(summary.wal_records, 8); - assert_eq!(summary.minimum_round, 8); - assert_eq!(summary.maximum_round, 11); - } - - #[test] - fn autonomous_clock_summary_requires_progress_after_the_benchmark_baseline() { - let metrics = vec![ - autonomous_clock_metrics(8, 0, 0), - autonomous_clock_metrics(8, 0, 0), - ]; - let baselines = metrics - .iter() - .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) - .collect::>(); - - assert!(!summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); - - for metrics in &metrics { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .inc(); - metrics.starfish_rbc_dag_shadow_carrier_round.inc(); - } - - assert!(summarize_autonomous_clock_benchmark(&metrics, 2, Some(&baselines)).verdict_valid); - } - - #[test] - fn autonomous_clock_summary_rejects_invalid_progress_and_unbounded_state() { - let no_progress = autonomous_clock_metrics(1, 0, 0); - no_progress - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .reset(); - let invalid_clock = autonomous_clock_metrics(12, 0, 0); - invalid_clock.starfish_rbc_dag_shadow_clock_valid.set(0); - let unbounded = autonomous_clock_metrics( - 20, - STARFISH_RBC_DAG_AUTONOMOUS_MAX_PHASE_BACKLOG_FACTOR * 4 + 1, - STARFISH_RBC_DAG_AUTONOMOUS_MAX_BUFFERED_FACTOR * 4 + 1, - ); - let metrics = vec![no_progress, invalid_clock, unbounded]; - - let summary = summarize_autonomous_clock_benchmark(&metrics, 4, None); - - assert!(!summary.verdict_valid); - assert_eq!(summary.valid_nodes, 2); - assert_eq!(summary.progress_nodes, 2); - assert_eq!(summary.bounded_nodes, 2); - assert!( - summary.maximum_round - summary.minimum_round - > STARFISH_RBC_DAG_AUTONOMOUS_MAX_ROUND_LAG - ); - } } diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index effc2889..f5342d8f 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -35,7 +35,7 @@ use crate::{ }, core::Core, core_thread::CoreThreadDispatcher, - crypto::{Blake3Hasher, BlsSigner, MacKey}, + crypto::{BlsSigner, MacKey}, dag_state::{ConsensusProtocol, DagState, DataSource}, data::Data, metrics::{Metrics, UtilizationTimerVecExt}, @@ -52,7 +52,6 @@ use crate::{ }, starfish_rbc_dag_shadow_service::{ ShadowServiceErrorV1, ShadowServiceEventV1, StarfishRbcDagShadowServiceHandleV1, - start_starfish_rbc_dag_autonomous_clock_service_v1, start_starfish_rbc_dag_shadow_service_v1, }, starfish_rbc_service::{ @@ -71,8 +70,6 @@ const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); const STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); -const STARFISH_RBC_DAG_AUTONOMOUS_INSTANCE_CONTEXT: &str = - "STARFISH_RBC_DAG_AUTONOMOUS_CLOCK_V1_PROTOCOL_INSTANCE"; /// Recover the exact locally selected Starfish-RBC chain so the persisted /// non-authoritative shadow can reconcile a WAL that ended before the direct @@ -142,40 +139,13 @@ fn recovered_local_rbc_headers( Ok(reversed) } -fn shadow_transport_error_invalidates_run(error: &ShadowServiceErrorV1) -> bool { +fn shadow_transport_error_invalidates_comparison(error: &ShadowServiceErrorV1) -> bool { matches!( error, ShadowServiceErrorV1::Overloaded { .. } | ShadowServiceErrorV1::Stopped ) } -fn invalidate_shadow_run(metrics: &Metrics) { - // Exactly one verdict is active for a configured shadow mode, but setting - // both to zero makes every transport/startup failure fail closed without - // duplicating mode knowledge throughout the network plumbing. - metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - metrics.starfish_rbc_dag_shadow_clock_valid.set(0); -} - -fn rbc_dag_shadow_protocol_instance( - direct_instance: [u8; 32], - autonomous_clock: bool, -) -> RbcDagProtocolInstanceId { - let bytes = if autonomous_clock { - // Autonomous heartbeat carriers intentionally do not authenticate in - // the same namespace as milestone-three's direct-header mirror. This - // makes a heterogeneous deployment fail closed at the shadow boundary - // instead of cross-admitting application and control carriers. - let mut hasher = Blake3Hasher::new_derive_key(STARFISH_RBC_DAG_AUTONOMOUS_INSTANCE_CONTEXT); - hasher.update(&direct_instance); - *hasher.finalize().as_bytes() - } else { - direct_instance - }; - RbcDagProtocolInstanceId::new(bytes) - .expect("a configured direct RBC instance and its derived namespace are nonzero") -} - /// Enforce the MAC experiment's transport contract before cryptographic /// verification: /// @@ -1105,8 +1075,8 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier(self.peer_id, envelope) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); + if shadow_transport_error_invalidates_comparison(&error) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); } tracing::warn!("Failed to forward RBC-DAG shadow carrier: {error}"); } @@ -1115,8 +1085,8 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier_request(self.peer_id, reference) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); + if shadow_transport_error_invalidates_comparison(&error) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); } tracing::warn!("Failed to forward RBC-DAG shadow request: {error}"); } @@ -1125,37 +1095,13 @@ impl ConnectionHandler { if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.carrier_response(self.peer_id, response) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); + if shadow_transport_error_invalidates_comparison(&error) { + self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); } tracing::warn!("Failed to forward RBC-DAG shadow response: {error}"); } } } - NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_sync_request(self.peer_id, request) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); - } - tracing::warn!( - "Failed to forward RBC-DAG shadow carrier sync request: {error}" - ); - } - } - } - NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_sync_response(self.peer_id, response) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&self.metrics); - } - tracing::warn!( - "Failed to forward RBC-DAG shadow carrier sync response: {error}" - ); - } - } - } } true } @@ -1807,13 +1753,7 @@ impl NetworkSyncer let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); - let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow - && node_parameters.starfish_rbc_dag_autonomous_clock - { - // Autonomous carrier rounds are independent of direct consensus - // rounds and recover entirely from their distinct WAL. - Some(Vec::new()) - } else if node_parameters.starfish_rbc_dag_shadow { + let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { match recovered_local_rbc_headers(&core) { Ok(headers) => Some(headers), Err(error) => { @@ -1906,10 +1846,8 @@ impl NetworkSyncer let protocol_instance_bytes = node_parameters .starfish_rbc_protocol_instance .expect("validated shadow configuration must share the direct RBC instance"); - let protocol_instance = rbc_dag_shadow_protocol_instance( - protocol_instance_bytes, - node_parameters.starfish_rbc_dag_autonomous_clock, - ); + let protocol_instance = RbcDagProtocolInstanceId::new(protocol_instance_bytes) + .expect("validated direct RBC instance must be nonzero"); let committee_context = RbcDagCommitteeContextV1::new(committee.clone()) .expect("validated committee must initialize the RBC-DAG shadow"); let context = RbcDagContextV1::new_with_committee( @@ -1932,40 +1870,19 @@ impl NetworkSyncer } }; // -1 means the background WAL replay has not completed yet; - // Ready moves the active observational mode to 1 unless work - // was already shed (0). The inactive verdict remains zero. - if node_parameters.starfish_rbc_dag_autonomous_clock { - metrics.starfish_rbc_dag_shadow_clock_valid.set(-1); - metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - } else { - metrics.starfish_rbc_dag_shadow_comparison_valid.set(-1); - metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - } - let started = if node_parameters.starfish_rbc_dag_autonomous_clock { - start_starfish_rbc_dag_autonomous_clock_service_v1( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - Duration::from_millis( - node_parameters.starfish_rbc_dag_heartbeat_interval_ms, - ), - ) - } else { - start_starfish_rbc_dag_shadow_service_v1( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - recovered_local_headers, - ) - }; - match started { + // Ready moves this to 1 unless work was already shed (0). + metrics.starfish_rbc_dag_shadow_comparison_valid.set(-1); + match start_starfish_rbc_dag_shadow_service_v1( + starfish_rbc_dag_shadow_wal, + committee_context, + dag_state.get_own_authority_index(), + context, + authorizer, + recovered_local_headers, + ) { Ok((service, events, task)) => (Some(service), Some(events), Some(task)), Err(error) => { - invalidate_shadow_run(&metrics); + metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); tracing::error!( "Disabling non-authoritative Starfish-RBC-DAG shadow: {error}" ); @@ -2168,8 +2085,10 @@ impl NetworkSyncer canonical.transactions_commitment(), ); if let Err(error) = shadow.direct_delivered(identity) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&rbc_metrics); + if shadow_transport_error_invalidates_comparison(&error) { + rbc_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); } tracing::warn!( "Failed to notify RBC-DAG shadow of direct delivery: {error}" @@ -2224,7 +2143,6 @@ impl NetworkSyncer shadow_metrics .starfish_rbc_dag_shadow_comparison_valid .set(0); - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } Err(mpsc::error::TrySendError::Closed(_)) => { shadow_metrics @@ -2234,7 +2152,6 @@ impl NetworkSyncer shadow_metrics .starfish_rbc_dag_shadow_comparison_valid .set(0); - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } } } else { @@ -2306,39 +2223,17 @@ impl NetworkSyncer .starfish_rbc_dag_shadow_wal_durable_records_total .inc_by(records); } - ShadowServiceEventV1::Ready { autonomous_clock } => { - let verdict = if autonomous_clock { - &shadow_metrics.starfish_rbc_dag_shadow_clock_valid - } else { - &shadow_metrics.starfish_rbc_dag_shadow_comparison_valid - }; - if verdict.get() != 0 { - verdict.set(1); + ShadowServiceEventV1::Ready => { + if shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .get() + != 0 + { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(1); } } - ShadowServiceEventV1::ClockState { - open_round, - phase_backlog, - admitted_authors, - admitted_stake, - buffered_authenticated, - } => { - shadow_metrics - .starfish_rbc_dag_shadow_carrier_round - .set(i64::from(open_round)); - shadow_metrics - .starfish_rbc_dag_shadow_phase_backlog - .set(i64::try_from(phase_backlog).unwrap_or(i64::MAX)); - shadow_metrics - .starfish_rbc_dag_shadow_admitted_authors - .set(i64::try_from(admitted_authors).unwrap_or(i64::MAX)); - shadow_metrics - .starfish_rbc_dag_shadow_admitted_stake - .set(i64::try_from(admitted_stake).unwrap_or(i64::MAX)); - shadow_metrics - .starfish_rbc_dag_shadow_buffered_authenticated - .set(i64::try_from(buffered_authenticated).unwrap_or(i64::MAX)); - } ShadowServiceEventV1::Recovered { batches, discarded_tail_bytes, @@ -2360,7 +2255,6 @@ impl NetworkSyncer shadow_metrics .starfish_rbc_dag_shadow_comparison_valid .set(0); - shadow_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); } tracing::warn!( "Rejected non-authoritative RBC-DAG shadow input from {:?}: {}", @@ -2925,8 +2819,10 @@ impl NetworkSyncer } if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.peer_connected(peer_id) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&shadow_metrics); + if shadow_transport_error_invalidates_comparison(&error) { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); } tracing::warn!( "Failed to notify RBC-DAG shadow that authority {} connected: {}", @@ -2984,8 +2880,10 @@ impl NetworkSyncer } if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { if let Err(error) = shadow.peer_disconnected(peer_id) { - if shadow_transport_error_invalidates_run(&error) { - invalidate_shadow_run(&shadow_metrics); + if shadow_transport_error_invalidates_comparison(&error) { + shadow_metrics + .starfish_rbc_dag_shadow_comparison_valid + .set(0); } tracing::warn!( "Failed to notify RBC-DAG shadow that authority {} disconnected: {}", @@ -3708,19 +3606,4 @@ mod tests { assert_eq!(unique.len(), selected.len()); assert!(selected.iter().all(|peer| candidates.contains(peer))); } - - #[test] - fn autonomous_carriers_use_a_distinct_authentication_namespace() { - let direct = [0x5A; 32]; - let mirror = rbc_dag_shadow_protocol_instance(direct, false); - let autonomous = rbc_dag_shadow_protocol_instance(direct, true); - - assert_eq!(mirror.as_bytes(), &direct); - assert_ne!(autonomous, mirror); - assert_eq!( - autonomous, - rbc_dag_shadow_protocol_instance(direct, true), - "derived autonomous namespace must be deterministic across nodes" - ); - } } diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 82805482..6531bd77 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -101,26 +101,6 @@ pub struct RbcDagShadowCarrierResponse { pub canonical_carrier: Vec, } -/// Request one exact carrier-clock slot from a peer. Keeping synchronization -/// slot-addressed prevents an untrusted peer from choosing an unbounded range -/// of history to return. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] -pub struct RbcDagShadowCarrierSyncRequest { - pub author: AuthorityIndex, - pub round: RoundNumber, -} - -/// Full response for one exact carrier-clock slot. The receiver validates that -/// the canonical carrier has the requested author and round, and authenticates -/// the sidecar before admitting it. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct RbcDagShadowCarrierSyncResponse { - pub author: AuthorityIndex, - pub round: RoundNumber, - pub canonical_carrier: Vec, - pub authentication_sidecar: Vec, -} - /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -236,12 +216,6 @@ pub enum NetworkMessage { RbcDagShadowCarrierRequest(BlockReference), /// Return content only; the receiver recomputes and checks the reference. RbcDagShadowCarrierResponse(RbcDagShadowCarrierResponse), - /// Starfish-RBC-DAG milestone-four synchronization for one exact - /// `(author, round)` carrier-clock slot. - RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest), - /// Full canonical carrier and authentication sidecar for an exact - /// carrier-clock slot. Receivers validate the duplicated slot identity. - RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse), } impl NetworkMessage { @@ -272,8 +246,6 @@ impl NetworkMessage { Self::RbcDagShadowCarrier(_) => "rbc_dag_shadow_carrier", Self::RbcDagShadowCarrierRequest(_) => "rbc_dag_shadow_carrier_request", Self::RbcDagShadowCarrierResponse(_) => "rbc_dag_shadow_carrier_response", - Self::RbcDagShadowCarrierSyncRequest(_) => "rbc_dag_shadow_carrier_sync_request", - Self::RbcDagShadowCarrierSyncResponse(_) => "rbc_dag_shadow_carrier_sync_response", } } } @@ -1180,18 +1152,6 @@ mod tests { reference: block_ref, canonical_carrier: vec![0xA6, 0xA7], }); - let sync_request = - NetworkMessage::RbcDagShadowCarrierSyncRequest(RbcDagShadowCarrierSyncRequest { - author: 2, - round: 23, - }); - let sync_response = - NetworkMessage::RbcDagShadowCarrierSyncResponse(RbcDagShadowCarrierSyncResponse { - author: 2, - round: 23, - canonical_carrier: vec![0xA8, 0xA9], - authentication_sidecar: vec![0xAA, 0xAB], - }); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), @@ -1201,8 +1161,6 @@ mod tests { (shadow, 15, "rbc_dag_shadow_carrier"), (shadow_request, 16, "rbc_dag_shadow_carrier_request"), (shadow_response, 17, "rbc_dag_shadow_carrier_response"), - (sync_request, 18, "rbc_dag_shadow_carrier_sync_request"), - (sync_response, 19, "rbc_dag_shadow_carrier_sync_response"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); @@ -1213,37 +1171,6 @@ mod tests { } } - #[test] - fn rbc_dag_shadow_carrier_sync_payloads_roundtrip_exactly() { - let request = RbcDagShadowCarrierSyncRequest { - author: 3, - round: 41, - }; - let encoded = - bincode::serialize(&NetworkMessage::RbcDagShadowCarrierSyncRequest(request)).unwrap(); - let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); - assert!(matches!( - decoded, - NetworkMessage::RbcDagShadowCarrierSyncRequest(decoded) if decoded == request - )); - - let response = RbcDagShadowCarrierSyncResponse { - author: 3, - round: 41, - canonical_carrier: vec![0xC1, 0xC2, 0xC3], - authentication_sidecar: vec![0xD1, 0xD2], - }; - let encoded = bincode::serialize(&NetworkMessage::RbcDagShadowCarrierSyncResponse( - response.clone(), - )) - .unwrap(); - let decoded: NetworkMessage = bincode::deserialize(&encoded).unwrap(); - assert!(matches!( - decoded, - NetworkMessage::RbcDagShadowCarrierSyncResponse(decoded) if decoded == response - )); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn scoped_connection_tasks_allow_immediate_same_port_rebind() { // Active sockets bind to listener_port * 10. Keep those derived ports diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs index 8d62a392..ba10e87f 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs @@ -34,7 +34,7 @@ use crate::{ }, types::{ AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, - RoundNumber, Stake, TimestampNs, + RoundNumber, TimestampNs, }, }; @@ -551,49 +551,6 @@ impl StarfishRbcDagShadowV1 { self.model.can_create_carrier() } - pub(crate) fn pending_phase_backlog_len(&self) -> usize { - self.model.pending_phase_backlog_len() - } - - pub(crate) fn admitted_reference( - &self, - authority: AuthorityIndex, - round: RoundNumber, - ) -> Option { - self.model.admitted_reference(authority, round) - } - - pub(crate) fn current_round_admitted_author_count(&self) -> usize { - let round = self.local_carrier_round(); - self.committee - .committee() - .authorities() - .filter(|authority| self.model.admitted_reference(*authority, round).is_some()) - .count() - } - - pub(crate) fn current_round_admitted_stake(&self) -> Stake { - let round = self.local_carrier_round(); - self.committee - .committee() - .authorities() - .filter(|authority| self.model.admitted_reference(*authority, round).is_some()) - .filter_map(|authority| self.committee.committee().get_stake(authority)) - .fold(0, Stake::saturating_add) - } - - /// Authenticated slots retained beyond the model's current admission - /// window. These are bounded by the reducer's future-carrier window and - /// become admitted only through sequential clock advancement. - pub(crate) fn buffered_authenticated_carrier_count(&self) -> usize { - self.authenticated_slots - .iter() - .filter(|((authority, round), reference)| { - self.model.admitted_reference(*authority, *round) != Some(**reference) - }) - .count() - } - pub(crate) fn wal_counts(&self) -> (u64, u64) { (self.wal.batch_count(), self.wal.record_count()) } @@ -651,17 +608,6 @@ impl StarfishRbcDagShadowV1 { Ok((envelope, effects)) } - /// Create the currently open autonomous control slot with no application - /// payload. The caller supplies only a timestamp: the round is derived - /// from the durable reducer and the empty commitment is canonical. - pub(crate) fn create_local_control_heartbeat( - &mut self, - creation_time_ns: TimestampNs, - ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { - let round = self.model.local_carrier_round(); - self.create_local_carrier(round, TransactionsCommitment::default(), creation_time_ns) - } - /// Verify and durably apply an authenticated network envelope for this /// exact receiver. #[cfg(test)] @@ -838,37 +784,6 @@ impl StarfishRbcDagShadowV1 { .map(<[u8]>::to_vec) } - /// Decode canonical carrier bytes without mutating the reducer. Sync - /// clients use this to bind a response to the requested author and round - /// before passing it through normal authenticated ingress. - pub(crate) fn candidate_slot( - &self, - canonical_carrier_wire: &[u8], - ) -> Result<(AuthorityIndex, RoundNumber, BlockReference), ShadowErrorV1> { - let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; - Ok(( - candidate.header().author(), - candidate.header().carrier_round(), - candidate.reference(), - )) - } - - /// Return the exact durably exposed local envelope for one carrier round. - /// Missing, incomplete, or unexposed slots are never served. - pub(crate) fn local_outbound_envelope( - &self, - round: RoundNumber, - ) -> Option { - let snapshot = self.journal.snapshot(); - let reference = snapshot.own_carrier(round)?; - let outbound = snapshot.outbound(reference)?; - outbound.exposed().then(|| ShadowOutboundEnvelopeV1 { - reference: outbound.reference(), - canonical_carrier_wire: outbound.canonical_carrier_wire().to_vec(), - authentication_sidecar: outbound.authentication_sidecar().to_vec(), - }) - } - /// Return every exposed local carrier in deterministic reference order. /// The same full sidecar is returned for every peer. pub(crate) fn retransmissions(&self) -> Vec { @@ -890,7 +805,7 @@ impl StarfishRbcDagShadowV1 { /// payload and creation timestamp before accepting new observations. pub(crate) fn local_outbound_metadata( &self, - ) -> Result, ShadowErrorV1> { + ) -> Result, ShadowErrorV1> { self.journal .snapshot() .retransmissions() @@ -905,8 +820,6 @@ impl StarfishRbcDagShadowV1 { candidate.header().carrier_round(), candidate.header().transactions_commitment(), candidate.header().creation_time_ns(), - candidate.header().data_acknowledgments().is_empty() - && candidate.header().consensus_vertex().is_none(), )) }) .collect() @@ -2140,139 +2053,6 @@ mod tests { ); } - #[test] - fn autonomous_control_heartbeat_derives_the_open_round_and_is_durably_addressable() { - let mut network = TestNetwork::new(); - let node = &mut network.nodes[0]; - assert_eq!(node.local_carrier_round(), 1); - assert_eq!(node.current_round_admitted_author_count(), 0); - assert_eq!(node.current_round_admitted_stake(), 0); - assert_eq!(node.pending_phase_backlog_len(), 0); - assert_eq!(node.buffered_authenticated_carrier_count(), 0); - assert_eq!(node.local_outbound_envelope(1), None); - - let before = node.wal_counts(); - let (heartbeat, effects) = node.create_local_control_heartbeat(123).unwrap(); - assert!(effects.is_empty()); - assert_eq!(node.wal_counts().0, before.0 + 1); - let candidate = decode_candidate( - heartbeat.canonical_carrier_wire(), - &network.committee, - Some(heartbeat.reference()), - ) - .unwrap(); - assert_eq!(candidate.header().author(), 0); - assert_eq!(candidate.header().carrier_round(), 1); - assert_eq!( - candidate.header().transactions_commitment(), - TransactionsCommitment::default() - ); - assert_eq!(candidate.header().creation_time_ns(), 123); - assert!(candidate.header().data_acknowledgments().is_empty()); - assert!(candidate.header().phase_batch().is_empty()); - assert!(candidate.header().consensus_vertex().is_none()); - assert_eq!(node.local_outbound_envelope(1), Some(heartbeat.clone())); - assert_eq!(node.local_outbound_envelope(2), None); - assert_eq!( - node.candidate_slot(heartbeat.canonical_carrier_wire()) - .unwrap(), - (0, 1, heartbeat.reference()) - ); - assert_eq!(node.admitted_reference(0, 1), Some(heartbeat.reference())); - assert_eq!(node.current_round_admitted_author_count(), 1); - assert_eq!(node.current_round_admitted_stake(), 1); - assert_eq!(node.pending_phase_backlog_len(), 1); - assert_eq!(node.buffered_authenticated_carrier_count(), 0); - - let durable_counts = node.wal_counts(); - assert!(matches!( - node.create_local_control_heartbeat(124), - Err(ShadowErrorV1::Model(ModelError::LocalCarrierAlreadyFixed( - 1 - ))) - )); - assert_eq!(node.wal_counts(), durable_counts); - - let mut trailing = heartbeat.canonical_carrier_wire().to_vec(); - trailing.push(0); - assert!(node.candidate_slot(&trailing).is_err()); - } - - #[test] - fn autonomous_control_heartbeat_advances_sequentially_and_reopens_exact_bytes() { - let mut network = TestNetwork::new(); - let first = network.nodes[0] - .create_local_control_heartbeat(1_000) - .unwrap() - .0; - for author in [1, 2] { - let candidate = round_one_candidate(author, &network.committee, 0x70 + author as u8); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: author, - keys: &network.keyrings[author as usize], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &candidate.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - author, - ) - .unwrap(); - } - assert_eq!(network.nodes[0].local_carrier_round(), 2); - assert!(network.nodes[0].can_create_carrier()); - assert_eq!(network.nodes[0].current_round_admitted_author_count(), 0); - - let second = network.nodes[0] - .create_local_control_heartbeat(2_000) - .unwrap() - .0; - let second_candidate = decode_candidate( - second.canonical_carrier_wire(), - &network.committee, - Some(second.reference()), - ) - .unwrap(); - assert_eq!(second_candidate.header().carrier_round(), 2); - assert_eq!(second_candidate.header().own_prev(), first.reference()); - assert_eq!( - second_candidate.header().transactions_commitment(), - TransactionsCommitment::default() - ); - assert_eq!( - network.nodes[0].local_outbound_envelope(1), - Some(first.clone()) - ); - assert_eq!( - network.nodes[0].local_outbound_envelope(2), - Some(second.clone()) - ); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let (restarted, report) = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap(); - assert!(report.replayed_batches() >= 4); - assert_eq!(restarted.local_carrier_round(), 2); - assert!(!restarted.can_create_carrier()); - assert_eq!(restarted.local_outbound_envelope(1), Some(first)); - assert_eq!(restarted.local_outbound_envelope(2), Some(second)); - } - #[test] fn authenticated_replays_and_slot_conflicts_do_not_grow_durable_state() { let mut network = TestNetwork::new(); diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs index 27787191..49879cc2 100644 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs @@ -8,11 +8,8 @@ use std::{ error::Error, fmt, path::Path, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + sync::Arc, + time::{Duration, Instant}, }; use parking_lot::Mutex; @@ -27,10 +24,7 @@ use tokio::{ use crate::{ crypto::{MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE}, - network::{ - NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse, - RbcDagShadowCarrierSyncRequest, RbcDagShadowCarrierSyncResponse, - }, + network::{NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse}, starfish_rbc::RbcCanonicalHeader, starfish_rbc_dag::{ MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, @@ -44,14 +38,13 @@ use crate::{ types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, }; -// A mirror run must absorb one complete committee fan-in plus a small reserve; -// autonomous repair additionally budgets a simultaneous request and response -// per peer. At the four-MiB carrier cap, allowing at most 64 queued inputs also -// caps carrier payload retention at 256 MiB (plus bounded sidecars and -// allocator overhead). This permits 60 mirror validators or 20 autonomous -// validators. Larger committees are rejected for this benchmark prototype -// instead of silently under-sizing the queue and reporting incomparable -// results. +// A shadow run must absorb one complete committee fan-in plus a small reserve +// for the local carrier and control notifications before its single fsync +// owner can drain. At the four-MiB carrier cap, allowing at most 64 queued +// inputs also caps carrier payload retention at 256 MiB (plus bounded +// sidecars and allocator overhead). Larger committees are rejected for this +// benchmark prototype instead of silently under-sizing the queue and +// reporting incomparable results. // Use the full bounded allowance even for a small committee. A single fan-in // reserve is insufficient when several round bursts arrive while the actor is // synchronously making the previous transition durable. @@ -59,42 +52,7 @@ const SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1: usize = 64; const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 64; const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; -const SHADOW_MAINTENANCE_INTERVAL_V1: Duration = Duration::from_millis(100); const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); -const SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1: Duration = Duration::from_millis(100); -const SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1: Duration = Duration::from_millis(500); - -/// Runtime role of the persisted carrier actor. -/// -/// Mirror mode preserves milestone three's one-to-one comparison against -/// direct Starfish-RBC headers. Autonomous mode opens an independent, -/// heartbeat-only carrier clock. It remains observational: neither mode can -/// call the core dispatcher or mutate authoritative consensus state. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ShadowServiceModeV1 { - DirectMirror, - AutonomousClock { heartbeat_interval: Duration }, -} - -impl ShadowServiceModeV1 { - fn is_autonomous(self) -> bool { - matches!(self, Self::AutonomousClock { .. }) - } - - fn heartbeat_interval(self) -> Option { - match self { - Self::DirectMirror => None, - Self::AutonomousClock { heartbeat_interval } => Some(heartbeat_interval), - } - } - - fn carrier_sync_grace_interval(self) -> Duration { - self.heartbeat_interval() - .map(|interval| interval.saturating_mul(2)) - .unwrap_or_default() - .max(SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1) - } -} #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct ShadowLocalCarrierV1 { @@ -129,18 +87,9 @@ enum ShadowServiceMessageV1 { peer: AuthorityIndex, response: RbcDagShadowCarrierResponse, }, - CarrierSyncRequest { - peer: AuthorityIndex, - request: RbcDagShadowCarrierSyncRequest, - }, - CarrierSyncResponse { - peer: AuthorityIndex, - response: RbcDagShadowCarrierSyncResponse, - }, DirectDeliveriesChanged, TopologyChanged, RetryRecovery, - HeartbeatTick, Shutdown(oneshot::Sender>), } @@ -151,7 +100,6 @@ pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { own_authority: AuthorityIndex, committee_size: usize, input_capacity: usize, - mode: ShadowServiceModeV1, desired_topology: Arc>>, desired_direct_deliveries: Arc>>, invalidated_by_overload: Arc>>, @@ -179,12 +127,6 @@ impl StarfishRbcDagShadowServiceHandleV1 { &self, header: &RbcCanonicalHeader, ) -> Result<(), ShadowServiceErrorV1> { - if self.mode.is_autonomous() { - // The autonomous carrier clock is deliberately independent from - // direct consensus rounds. Direct headers continue through the - // authoritative path and cannot consume carrier slots. - return Ok(()); - } self.send(ShadowServiceMessageV1::LocalCarrier( ShadowLocalCarrierV1::from_direct_header(header), )) @@ -229,45 +171,10 @@ impl StarfishRbcDagShadowServiceHandleV1 { self.send(ShadowServiceMessageV1::CarrierResponse { peer, response }) } - pub(crate) fn carrier_sync_request( - &self, - peer: AuthorityIndex, - request: RbcDagShadowCarrierSyncRequest, - ) -> Result<(), ShadowServiceErrorV1> { - if !self.mode.is_autonomous() { - return Ok(()); - } - self.send(ShadowServiceMessageV1::CarrierSyncRequest { peer, request }) - } - - pub(crate) fn carrier_sync_response( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierSyncResponse, - ) -> Result<(), ShadowServiceErrorV1> { - if !self.mode.is_autonomous() { - return Ok(()); - } - validate_wire_size( - "carrier sync response", - response.canonical_carrier.len(), - MAX_CARRIER_CONTENT_SIZE_V1, - )?; - validate_wire_size( - "carrier sync authentication sidecar", - response.authentication_sidecar.len(), - self.max_sidecar_size, - )?; - self.send(ShadowServiceMessageV1::CarrierSyncResponse { peer, response }) - } - pub(crate) fn direct_delivered( &self, identity: ShadowDeliveryIdentityV1, ) -> Result<(), ShadowServiceErrorV1> { - if self.mode.is_autonomous() { - return Ok(()); - } if identity.author as usize >= self.committee_size { return Err(ShadowServiceErrorV1::UnknownAuthority(identity.author)); } @@ -338,12 +245,9 @@ impl ShadowServiceMessageV1 { Self::Carrier { .. } => "carrier", Self::CarrierRequest { .. } => "carrier_request", Self::CarrierResponse { .. } => "carrier_response", - Self::CarrierSyncRequest { .. } => "carrier_sync_request", - Self::CarrierSyncResponse { .. } => "carrier_sync_response", Self::DirectDeliveriesChanged => "direct_deliveries_changed", Self::TopologyChanged => "topology_changed", Self::RetryRecovery => "recovery_retry", - Self::HeartbeatTick => "heartbeat_tick", Self::Shutdown(_) => "shutdown", } } @@ -351,16 +255,7 @@ impl ShadowServiceMessageV1 { #[derive(Debug)] pub(crate) enum ShadowServiceEventV1 { - Ready { - autonomous_clock: bool, - }, - ClockState { - open_round: RoundNumber, - phase_backlog: usize, - admitted_authors: usize, - admitted_stake: u64, - buffered_authenticated: usize, - }, + Ready, ComparisonBacklog { unpaired_direct: usize, unpaired_shadow: usize, @@ -410,7 +305,6 @@ pub(crate) enum ShadowServiceErrorV1 { }, CommitteeBurstTooLarge { committee_size: usize, - required_capacity: usize, maximum_capacity: usize, }, UnknownAuthority(AuthorityIndex), @@ -418,7 +312,6 @@ pub(crate) enum ShadowServiceErrorV1 { ConflictingLocalHeader(RoundNumber), MissingRecoveredLocalHeader(RoundNumber), RecoveredLocalHeaderMismatch(RoundNumber), - AutonomousWalContainsApplicationCarrier(RoundNumber), LocalHeaderAuthority { expected: AuthorityIndex, actual: AuthorityIndex, @@ -429,21 +322,6 @@ pub(crate) enum ShadowServiceErrorV1 { peer: AuthorityIndex, reference: BlockReference, }, - InvalidHeartbeatInterval, - SyncRequestForForeignAuthor { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - UnexpectedSyncResponse { - author: AuthorityIndex, - round: RoundNumber, - }, - SyncResponseSlotMismatch { - expected_author: AuthorityIndex, - expected_round: RoundNumber, - actual_author: AuthorityIndex, - actual_round: RoundNumber, - }, } impl fmt::Display for ShadowServiceErrorV1 { @@ -474,12 +352,14 @@ impl fmt::Display for ShadowServiceErrorV1 { ), Self::CommitteeBurstTooLarge { committee_size, - required_capacity, maximum_capacity, } => write!( formatter, "Starfish-RBC-DAG shadow committee size {committee_size} needs a burst queue of \ - {required_capacity}, above the memory-safe capacity limit {maximum_capacity}", + {}, above the memory-safe capacity limit {maximum_capacity}", + committee_size + .saturating_sub(1) + .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1), ), Self::UnknownAuthority(authority) => { write!(formatter, "unknown shadow peer authority {authority}") @@ -499,10 +379,6 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "persisted shadow carrier and recovered direct header disagree at round {round}" ), - Self::AutonomousWalContainsApplicationCarrier(round) => write!( - formatter, - "autonomous carrier-clock WAL contains a non-heartbeat local carrier at round {round}" - ), Self::LocalHeaderAuthority { expected, actual } => write!( formatter, "shadow local header authority {actual} does not match local authority {expected}" @@ -517,26 +393,6 @@ impl fmt::Display for ShadowServiceErrorV1 { formatter, "shadow response for {reference} came from non-holder {peer}" ), - Self::InvalidHeartbeatInterval => formatter.write_str( - "Starfish-RBC-DAG autonomous heartbeat interval must be nonzero", - ), - Self::SyncRequestForForeignAuthor { expected, actual } => write!( - formatter, - "shadow carrier sync request asked authority {expected} to serve authority {actual}" - ), - Self::UnexpectedSyncResponse { author, round } => write!( - formatter, - "unexpected shadow carrier sync response for authority {author} round {round}" - ), - Self::SyncResponseSlotMismatch { - expected_author, - expected_round, - actual_author, - actual_round, - } => write!( - formatter, - "shadow carrier sync response for authority {expected_author} round {expected_round} contained authority {actual_author} round {actual_round}" - ), } } } @@ -571,65 +427,9 @@ pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( JoinHandle<()>, ), ShadowServiceErrorV1, -> { - start_starfish_rbc_dag_shadow_service_with_mode_v1( - path, - committee, - own_authority, - context, - authorizer, - recovered_local_headers, - ShadowServiceModeV1::DirectMirror, - ) -} - -pub(crate) fn start_starfish_rbc_dag_autonomous_clock_service_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - heartbeat_interval: Duration, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, -> { - if heartbeat_interval.is_zero() { - return Err(ShadowServiceErrorV1::InvalidHeartbeatInterval); - } - start_starfish_rbc_dag_shadow_service_with_mode_v1( - path, - committee, - own_authority, - context, - authorizer, - Vec::new(), - ShadowServiceModeV1::AutonomousClock { heartbeat_interval }, - ) -} - -fn start_starfish_rbc_dag_shadow_service_with_mode_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, - mode: ShadowServiceModeV1, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, > { let committee_size = committee.committee().len(); - let input_capacity = shadow_input_capacity(committee_size, mode)?; + let input_capacity = shadow_input_capacity(committee_size)?; let max_sidecar_size = authentication_sidecar_size(context.authentication_scheme(), committee_size); let path = path.as_ref().to_path_buf(); @@ -653,12 +453,9 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); let invalidated_by_overload = Arc::new(Mutex::new(None)); - let retry_notification_pending = Arc::new(AtomicBool::new(false)); - let heartbeat_notification_pending = Arc::new(AtomicBool::new(false)); let retry_tx = message_tx.downgrade(); - let retry_pending = Arc::clone(&retry_notification_pending); tokio::spawn(async move { - let mut interval = tokio::time::interval(SHADOW_MAINTENANCE_INTERVAL_V1); + let mut interval = tokio::time::interval(SHADOW_RECOVERY_RETRY_INTERVAL_V1); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); interval.tick().await; loop { @@ -666,56 +463,16 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let Some(retry_tx) = retry_tx.upgrade() else { break; }; - if retry_pending.swap(true, Ordering::AcqRel) { - continue; - } match retry_tx.try_send(ShadowServiceMessageV1::RetryRecovery) { - Ok(()) => {} - Err(TrySendError::Full(_)) => retry_pending.store(false, Ordering::Release), - Err(TrySendError::Closed(_)) => { - retry_pending.store(false, Ordering::Release); - break; - } + Ok(()) | Err(TrySendError::Full(_)) => {} + Err(TrySendError::Closed(_)) => break, } } }); - if let Some(heartbeat_interval) = mode.heartbeat_interval() { - let heartbeat_tx = message_tx.downgrade(); - let heartbeat_pending = Arc::clone(&heartbeat_notification_pending); - tokio::spawn(async move { - let mut interval = tokio::time::interval(heartbeat_interval); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // Give startup/WAL replay one full interval before the first - // carrier. A missed/full notification is harmless: a later tick - // retries the still-open local slot. - interval.tick().await; - loop { - interval.tick().await; - let Some(heartbeat_tx) = heartbeat_tx.upgrade() else { - break; - }; - if heartbeat_pending.swap(true, Ordering::AcqRel) { - continue; - } - match heartbeat_tx.try_send(ShadowServiceMessageV1::HeartbeatTick) { - Ok(()) => {} - Err(TrySendError::Full(_)) => { - heartbeat_pending.store(false, Ordering::Release); - } - Err(TrySendError::Closed(_)) => { - heartbeat_pending.store(false, Ordering::Release); - break; - } - } - } - }); - } let startup_events = event_tx.clone(); let actor_desired_topology = Arc::clone(&desired_topology); let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); - let actor_retry_notification_pending = Arc::clone(&retry_notification_pending); - let actor_heartbeat_notification_pending = Arc::clone(&heartbeat_notification_pending); let task = tokio::spawn(async move { let opened = tokio::task::spawn_blocking(move || { StarfishRbcDagShadowV1::open(path, committee, own_authority, context, authorizer) @@ -745,8 +502,8 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( let persisted_local = match core.local_outbound_metadata() { Ok(metadata) => metadata .into_iter() - .map(|(round, commitment, creation_time_ns, control_shape)| { - (round, (commitment, creation_time_ns, control_shape)) + .map(|(round, commitment, creation_time_ns)| { + (round, (commitment, creation_time_ns)) }) .collect::>(), Err(error) => { @@ -760,66 +517,43 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( } }; let durable_round = core.local_carrier_round(); - if mode.is_autonomous() { - if let Some((round, _)) = - persisted_local - .iter() - .find(|(_, (commitment, _, control_shape))| { - *commitment != crate::crypto::TransactionsCommitment::default() - || !*control_shape - }) - { + for (round, (commitment, creation_time_ns)) in &persisted_local { + let Some(recovered) = pending_local.get(round) else { let _ = startup_events .send(ShadowServiceEventV1::Rejected { peer: None, - error: ShadowServiceErrorV1::AutonomousWalContainsApplicationCarrier( - *round, - ) - .to_string(), + error: ShadowServiceErrorV1::MissingRecoveredLocalHeader(*round) + .to_string(), }) .await; return; - } - } else { - for (round, (commitment, creation_time_ns, _)) in &persisted_local { - let Some(recovered) = pending_local.get(round) else { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::MissingRecoveredLocalHeader(*round) - .to_string(), - }) - .await; - return; - }; - if recovered.transactions_commitment != *commitment - || recovered.creation_time_ns != *creation_time_ns - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) - .to_string(), - }) - .await; - return; - } - } - if let Some(round) = pending_local - .keys() - .copied() - .find(|round| *round < durable_round && !persisted_local.contains_key(round)) + }; + if recovered.transactions_commitment != *commitment + || recovered.creation_time_ns != *creation_time_ns { let _ = startup_events .send(ShadowServiceEventV1::Rejected { peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(round) + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) .to_string(), }) .await; return; } } + if let Some(round) = pending_local + .keys() + .copied() + .find(|round| *round < durable_round && !persisted_local.contains_key(round)) + { + let _ = startup_events + .send(ShadowServiceEventV1::Rejected { + peer: None, + error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(round).to_string(), + }) + .await; + return; + } let reported_shadow_deliveries = match core.delivered_identities() { Ok(identities) => identities.into_iter().collect::>(), Err(error) => { @@ -839,10 +573,8 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( .collect(); let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); pending_local.retain(|round, _| *round >= core.local_carrier_round()); - let sync_round = core.local_carrier_round(); let state = ShadowServiceStateV1 { core, - mode, own_authority, committee_size, events: event_tx, @@ -854,14 +586,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( pending_local, pending_recovery: BTreeMap::new(), recovery_last_attempt: BTreeMap::new(), - sync_last_attempt: BTreeMap::new(), - sync_last_served: BTreeMap::new(), - sync_round, - sync_round_opened_at: Instant::now(), - sync_catch_up: false, - sync_used_in_open_round: false, - retry_notification_pending: actor_retry_notification_pending, - heartbeat_notification_pending: actor_heartbeat_notification_pending, direct_deliveries: BTreeSet::new(), reported_shadow_deliveries, recovered_shadow_deliveries, @@ -891,7 +615,6 @@ fn start_starfish_rbc_dag_shadow_service_with_mode_v1( own_authority, committee_size, input_capacity, - mode, desired_topology, desired_direct_deliveries, invalidated_by_overload, @@ -967,7 +690,6 @@ impl ShadowComparisonBacklogV1 { struct ShadowServiceStateV1 { core: StarfishRbcDagShadowV1, - mode: ShadowServiceModeV1, own_authority: AuthorityIndex, committee_size: usize, events: mpsc::Sender, @@ -979,14 +701,6 @@ struct ShadowServiceStateV1 { pending_local: BTreeMap, pending_recovery: BTreeMap>, recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, - sync_last_attempt: BTreeMap<(AuthorityIndex, RoundNumber), Instant>, - sync_last_served: BTreeMap, - sync_round: RoundNumber, - sync_round_opened_at: Instant, - sync_catch_up: bool, - sync_used_in_open_round: bool, - retry_notification_pending: Arc, - heartbeat_notification_pending: Arc, direct_deliveries: BTreeSet, reported_shadow_deliveries: BTreeSet, recovered_shadow_deliveries: BTreeSet, @@ -1010,9 +724,6 @@ impl ShadowServiceStateV1 { } fn emit_comparison_backlog(&self) { - if self.mode.is_autonomous() { - return; - } let (unpaired_direct, unpaired_shadow, max_round_lag) = self.comparison_backlog.counts(); self.emit(ShadowServiceEventV1::ComparisonBacklog { unpaired_direct, @@ -1021,19 +732,6 @@ impl ShadowServiceStateV1 { }); } - fn emit_clock_state(&self) { - if !self.mode.is_autonomous() { - return; - } - self.emit(ShadowServiceEventV1::ClockState { - open_round: self.core.local_carrier_round(), - phase_backlog: self.core.pending_phase_backlog_len(), - admitted_authors: self.core.current_round_admitted_author_count(), - admitted_stake: self.core.current_round_admitted_stake(), - buffered_authenticated: self.core.buffered_authenticated_carrier_count(), - }); - } - fn validate_peer(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { if peer as usize >= self.committee_size { return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); @@ -1053,10 +751,6 @@ impl ShadowServiceStateV1 { } self.recovery_last_attempt .retain(|(_, holder), _| holder != peer); - self.sync_last_attempt - .retain(|(author, _), _| author != peer); - self.sync_last_served - .retain(|requester, _| requester != peer); if state.0 { self.connected.insert(*peer); newly_connected.push(*peer); @@ -1066,17 +760,10 @@ impl ShadowServiceStateV1 { } self.observed_topology = desired; if !newly_connected.is_empty() { - if self.mode.is_autonomous() { - // Autonomous history is synchronized one exact slot at a - // time. Replaying the entire retained run on every reconnect - // would create an unbounded burst as heartbeats accumulate. - self.flush_carrier_sync_requests(self.core.local_carrier_round() > 1); - } else { - let retransmissions = self.core.retransmissions(); - for peer in newly_connected { - for envelope in &retransmissions { - self.send_envelope(peer, envelope); - } + let retransmissions = self.core.retransmissions(); + for peer in newly_connected { + for envelope in &retransmissions { + self.send_envelope(peer, envelope); } } self.flush_recovery_requests(); @@ -1084,9 +771,6 @@ impl ShadowServiceStateV1 { } fn reconcile_direct_deliveries(&mut self) { - if self.mode.is_autonomous() { - return; - } let desired = self.desired_direct_deliveries.lock().clone(); let newly_observed = desired .difference(&self.direct_deliveries) @@ -1161,12 +845,6 @@ impl ShadowServiceStateV1 { } fn process_effects(&mut self, effects: Vec) { - let carrier_round_advanced = effects - .iter() - .any(|effect| matches!(effect, ModelEffect::CarrierRoundAdvanced(_))); - if carrier_round_advanced { - self.sync_catch_up = std::mem::take(&mut self.sync_used_in_open_round); - } for effect in effects { match effect { ModelEffect::NeedCarrier { target, holders } => { @@ -1180,8 +858,7 @@ impl ShadowServiceStateV1 { ModelEffect::Delivered(reference) => { self.pending_recovery.remove(&reference); } - ModelEffect::PrefixAdvanced { .. } => {} - ModelEffect::CarrierRoundAdvanced(_) => {} + ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} } } self.reconcile_pending_recovery(); @@ -1190,59 +867,6 @@ impl ShadowServiceStateV1 { self.pending_recovery.len(), )); self.report_new_shadow_deliveries(); - self.flush_carrier_sync_requests(false); - self.emit_clock_state(); - } - - fn try_create_autonomous_heartbeat(&mut self) { - if !self.mode.is_autonomous() || !self.core.can_create_carrier() { - self.emit_clock_state(); - return; - } - let creation_time_ns = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - .try_into() - .unwrap_or(TimestampNs::MAX); - let before = self.core.wal_counts(); - match self.core.create_local_control_heartbeat(creation_time_ns) { - Ok((envelope, effects)) => { - self.emit(ShadowServiceEventV1::Input { - kind: "heartbeat", - outcome: "accepted", - }); - self.report_wal_delta(before); - self.broadcast(&envelope); - self.process_effects(effects); - } - Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { - // The local slot is open syntactically but cannot yet name a - // quorum of exact previous-round admitted parents. A later - // authenticated ingress or timer tick retries it. - self.emit(ShadowServiceEventV1::Input { - kind: "heartbeat", - outcome: "waiting_for_quorum", - }); - self.emit_clock_state(); - } - Err(error) => self.mark_fatal(error), - } - } - - /// While repairing a lagging clock, fix our next control carrier as soon - /// as its exact previous-round quorum is available. Waiting for the - /// normal heartbeat interval would cap catch-up at the production rate, - /// so a node behind a continuously advancing committee could never close - /// the gap. Healthy rounds still remain paced exclusively by the timer. - fn drive_autonomous_catch_up(&mut self) { - while self.sync_catch_up && self.core.can_create_carrier() && !self.fatal { - let round_before = self.core.local_carrier_round(); - self.try_create_autonomous_heartbeat(); - if self.core.local_carrier_round() == round_before { - break; - } - } } fn reconcile_pending_recovery(&mut self) { @@ -1376,231 +1000,6 @@ impl ShadowServiceStateV1 { } } - fn flush_carrier_sync_requests(&mut self, force: bool) { - if !self.mode.is_autonomous() { - return; - } - let round = self.core.local_carrier_round(); - let now = Instant::now(); - if round != self.sync_round { - self.sync_round = round; - self.sync_round_opened_at = now; - self.sync_last_attempt.clear(); - } - self.sync_last_attempt.retain(|(author, attempt_round), _| { - *attempt_round == round - && self.connected.contains(author) - && self.core.admitted_reference(*author, round).is_none() - }); - if !force - && !self.sync_catch_up - && now.saturating_duration_since(self.sync_round_opened_at) - < self.mode.carrier_sync_grace_interval() - { - return; - } - let requests = self - .connected - .iter() - .copied() - .filter(|author| self.core.admitted_reference(*author, round).is_none()) - .filter(|author| { - self.sync_last_attempt - .get(&(*author, round)) - .is_none_or(|last| { - now.saturating_duration_since(*last) - >= SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 - }) - }) - .collect::>(); - for author in requests { - self.sync_last_attempt.insert((author, round), now); - self.emit(ShadowServiceEventV1::Network { - recipient: author, - message: NetworkMessage::RbcDagShadowCarrierSyncRequest( - RbcDagShadowCarrierSyncRequest { author, round }, - ), - }); - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "sent", - }); - } - } - - fn handle_carrier_sync_request( - &mut self, - peer: AuthorityIndex, - request: RbcDagShadowCarrierSyncRequest, - ) { - if request.author != self.own_authority { - self.reject( - Some(peer), - ShadowServiceErrorV1::SyncRequestForForeignAuthor { - expected: self.own_authority, - actual: request.author, - }, - ); - return; - } - if request.round > self.core.local_carrier_round() { - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "future_not_found", - }); - return; - } - let now = Instant::now(); - if self.sync_last_served.get(&peer).is_some_and(|(_, last)| { - now.saturating_duration_since(*last) < SHADOW_CARRIER_SYNC_RETRY_INTERVAL_V1 - }) { - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "rate_limited", - }); - return; - } - self.sync_last_served.insert(peer, (request.round, now)); - let Some(envelope) = self.core.local_outbound_envelope(request.round) else { - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "not_found", - }); - return; - }; - self.emit(ShadowServiceEventV1::Network { - recipient: peer, - message: NetworkMessage::RbcDagShadowCarrierSyncResponse( - RbcDagShadowCarrierSyncResponse { - author: request.author, - round: request.round, - canonical_carrier: envelope.canonical_carrier_wire().to_vec(), - authentication_sidecar: envelope.authentication_sidecar().to_vec(), - }, - ), - }); - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "served", - }); - } - - fn handle_carrier_sync_response( - &mut self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierSyncResponse, - ) { - let expected = (response.author, response.round); - if peer != response.author { - self.reject( - Some(peer), - ShadowServiceErrorV1::UnexpectedSyncResponse { - author: response.author, - round: response.round, - }, - ); - return; - } - let (actual_author, actual_round, actual_reference) = - match self.core.candidate_slot(&response.canonical_carrier) { - Ok(slot) => slot, - Err(error) => { - self.reject(Some(peer), error); - return; - } - }; - if actual_author != response.author || actual_round != response.round { - self.reject( - Some(peer), - ShadowServiceErrorV1::SyncResponseSlotMismatch { - expected_author: response.author, - expected_round: response.round, - actual_author, - actual_round, - }, - ); - return; - } - if response.round < self.core.local_carrier_round() - || self - .core - .admitted_reference(response.author, response.round) - .is_some() - { - self.sync_last_attempt.remove(&expected); - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_response", - outcome: "ignored_already_admitted_or_stale", - }); - return; - } - if !self.sync_last_attempt.contains_key(&expected) { - self.reject( - Some(peer), - ShadowServiceErrorV1::UnexpectedSyncResponse { - author: response.author, - round: response.round, - }, - ); - return; - } - let before = self.core.wal_counts(); - match self.core.receive_or_retain_from_peer( - &response.canonical_carrier, - &response.authentication_sidecar, - peer, - ) { - Ok(outcome) => { - let outcome_label = match outcome.disposition() { - ShadowIngressDispositionV1::Authenticated => "authenticated", - ShadowIngressDispositionV1::CandidateRetained => "retained_unauthenticated", - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => "ignored", - }; - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_response", - outcome: outcome_label, - }); - if outcome.disposition() == ShadowIngressDispositionV1::Authenticated - && self - .core - .admitted_reference(response.author, response.round) - == Some(actual_reference) - { - self.sync_used_in_open_round = true; - } - self.report_wal_delta(before); - self.process_effects(outcome.effects().to_vec()); - if self - .core - .admitted_reference(response.author, response.round) - .is_some() - { - self.sync_last_attempt.remove(&expected); - } - if self.sync_catch_up { - self.flush_carrier_sync_requests(true); - } - if outcome.disposition() == ShadowIngressDispositionV1::CandidateRetained { - self.reject( - Some(peer), - ShadowServiceErrorV1::UnauthenticatedCarrierRetained, - ); - } - } - Err(error) => { - self.emit(ShadowServiceEventV1::Input { - kind: "carrier_sync_response", - outcome: "rejected", - }); - if is_fatal_core_error(&error) { - self.mark_fatal(error); - } else { - self.reject(Some(peer), error); - } - } - } - } - fn report_new_shadow_deliveries(&mut self) { let identities: BTreeSet<_> = match self.core.delivered_identities() { Ok(identities) => identities.into_iter().collect(), @@ -1616,9 +1015,7 @@ impl ShadowServiceStateV1 { self.reported_shadow_deliveries = identities; for identity in &new_identities { let slot = delivery_slot(identity); - if !self.mode.is_autonomous() { - self.comparison_backlog.observe_epoch_shadow(slot); - } + self.comparison_backlog.observe_epoch_shadow(slot); self.emit(ShadowServiceEventV1::Delivered(*identity)); self.emit_slot_comparison(slot); self.emit_comparison_backlog(); @@ -1626,9 +1023,6 @@ impl ShadowServiceStateV1 { } fn emit_slot_comparison(&mut self, slot: ShadowDeliverySlotV1) { - if self.mode.is_autonomous() { - return; - } let direct = self .direct_deliveries .iter() @@ -1691,16 +1085,10 @@ fn run_shadow_service( state.reconcile_topology(); state.reconcile_direct_deliveries(); if !state.observe_external_invalidation() { - state.emit(ShadowServiceEventV1::Ready { - autonomous_clock: state.mode.is_autonomous(), - }); + state.emit(ShadowServiceEventV1::Ready); state.emit_comparison_backlog(); state.process_effects(open_report.recovery_effects().to_vec()); - if state.mode.is_autonomous() { - state.emit_clock_state(); - } else { - state.retry_pending_local(); - } + state.retry_pending_local(); } while !state.fatal { @@ -1728,15 +1116,6 @@ fn run_shadow_service( if state.observe_external_invalidation() { break; } - match &message { - ShadowServiceMessageV1::RetryRecovery => state - .retry_notification_pending - .store(false, Ordering::Release), - ShadowServiceMessageV1::HeartbeatTick => state - .heartbeat_notification_pending - .store(false, Ordering::Release), - _ => {} - } match message { ShadowServiceMessageV1::LocalCarrier(local) => { state.enqueue_local(local); @@ -1811,17 +1190,6 @@ fn run_shadow_service( state.reject(Some(peer), error); continue; } - if state - .core - .retained_candidate_wire(response.reference) - .is_some() - { - state.emit(ShadowServiceEventV1::Input { - kind: "recovery", - outcome: "ignored_already_retained", - }); - continue; - } let Some(holders) = state.pending_recovery.get(&response.reference) else { state.reject( Some(peer), @@ -1870,20 +1238,6 @@ fn run_shadow_service( } } } - ShadowServiceMessageV1::CarrierSyncRequest { peer, request } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - state.handle_carrier_sync_request(peer, request); - } - ShadowServiceMessageV1::CarrierSyncResponse { peer, response } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - state.handle_carrier_sync_response(peer, response); - } ShadowServiceMessageV1::DirectDeliveriesChanged => { state.reconcile_direct_deliveries(); } @@ -1892,15 +1246,11 @@ fn run_shadow_service( state.reconcile_topology(); state.reconcile_pending_recovery(); state.flush_recovery_requests(); - state.flush_carrier_sync_requests(false); } - ShadowServiceMessageV1::HeartbeatTick => state.try_create_autonomous_heartbeat(), ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), } state.reconcile_topology(); state.reconcile_direct_deliveries(); - state.drive_autonomous_catch_up(); - state.flush_carrier_sync_requests(false); } let events = state.events.clone(); if let Err(error) = state.core.shutdown() { @@ -1929,19 +1279,13 @@ fn authentication_sidecar_size(scheme: BlockAuthenticationScheme, committee_size } } -fn shadow_input_capacity( - committee_size: usize, - mode: ShadowServiceModeV1, -) -> Result { - let peer_count = committee_size.saturating_sub(1); - let peer_burst_factor = if mode.is_autonomous() { 3 } else { 1 }; - let committee_burst = peer_count - .saturating_mul(peer_burst_factor) +fn shadow_input_capacity(committee_size: usize) -> Result { + let committee_burst = committee_size + .saturating_sub(1) .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1); if committee_burst > SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1 { return Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { committee_size, - required_capacity: committee_burst, maximum_capacity: SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1, }); } @@ -2005,11 +1349,7 @@ mod tests { impl Harness { fn new() -> Self { - Self::new_with_n(N) - } - - fn new_with_n(n: usize) -> Self { - let committee = Committee::new_test(vec![1; n]); + let committee = Committee::new_test(vec![1; N]); let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); let context = RbcDagContextV1::new_with_committee( RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), @@ -2017,14 +1357,14 @@ mod tests { BlockAuthenticationScheme::MacVector, ); let directory = tempfile::tempdir().unwrap(); - let paths = (0..n) + let paths = (0..N) .map(|authority| directory.path().join(format!("shadow-{authority}.wal"))) .collect(); Self { _directory: directory, committee, context, - keyrings: mac_keyrings_for_test(n), + keyrings: mac_keyrings_for_test(N), paths, } } @@ -2049,37 +1389,6 @@ mod tests { .unwrap() } - fn start_autonomous( - &self, - authority: AuthorityIndex, - ) -> ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ) { - self.start_autonomous_with_interval(authority, Duration::from_secs(60 * 60)) - } - - fn start_autonomous_with_interval( - &self, - authority: AuthorityIndex, - heartbeat_interval: Duration, - ) -> ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ) { - start_starfish_rbc_dag_autonomous_clock_service_v1( - &self.paths[authority as usize], - self.committee.clone(), - authority, - self.context, - ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), - heartbeat_interval, - ) - .unwrap() - } - fn envelope( &self, candidate: &CandidateCarrierV1, @@ -2113,7 +1422,7 @@ mod tests { async fn wait_ready(events: &mut mpsc::Receiver) { loop { match next_event(events).await { - ShadowServiceEventV1::Ready { .. } => return, + ShadowServiceEventV1::Ready => return, ShadowServiceEventV1::Rejected { error, .. } => { panic!("shadow startup failed: {error}") } @@ -2158,175 +1467,6 @@ mod tests { } } - async fn pump_autonomous_until_round( - handles: &[StarfishRbcDagShadowServiceHandleV1], - events: &mut [mpsc::Receiver], - open_rounds: &mut [RoundNumber], - deliveries: &mut [usize], - sync_requests: &mut usize, - target_open_round: RoundNumber, - ) { - timeout(EVENT_TIMEOUT, async { - loop { - let mut progressed = false; - for sender in 0..events.len() { - while let Ok(event) = events[sender].try_recv() { - progressed = true; - match event { - ShadowServiceEventV1::Network { recipient, message } => { - let recipient = recipient as usize; - match message { - NetworkMessage::RbcDagShadowCarrier(envelope) => handles - [recipient] - .carrier(sender as AuthorityIndex, envelope) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierRequest(reference) => handles - [recipient] - .carrier_request(sender as AuthorityIndex, reference) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierResponse(response) => handles - [recipient] - .carrier_response(sender as AuthorityIndex, response) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { - *sync_requests = sync_requests.saturating_add(1); - handles[recipient] - .carrier_sync_request( - sender as AuthorityIndex, - request, - ) - .unwrap(); - } - NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { - handles[recipient] - .carrier_sync_response( - sender as AuthorityIndex, - response, - ) - .unwrap(); - } - unexpected => panic!( - "autonomous shadow emitted unexpected network message: {unexpected:?}" - ), - } - } - ShadowServiceEventV1::ClockState { open_round, .. } => { - open_rounds[sender] = open_rounds[sender].max(open_round); - } - ShadowServiceEventV1::Delivered(_) => { - deliveries[sender] = deliveries[sender].saturating_add(1); - } - ShadowServiceEventV1::Rejected { error, .. } - if error.contains("FutureCarrierOutsideBuffer") - || error.contains("unexpected shadow response") => {} - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("autonomous shadow rejected valid test traffic: {error}") - } - _ => {} - } - } - } - if open_rounds - .iter() - .all(|round| *round >= target_open_round) - { - return; - } - if !progressed { - tokio::time::sleep(Duration::from_millis(1)).await; - } else { - tokio::task::yield_now().await; - } - } - }) - .await - .unwrap_or_else(|_| { - panic!( - "autonomous clock did not open round {target_open_round}; open={open_rounds:?}, sync_requests={sync_requests}" - ) - }); - } - - async fn pump_online_prefix_until_round( - handles: &[StarfishRbcDagShadowServiceHandleV1], - events: &mut [mpsc::Receiver], - open_rounds: &mut [RoundNumber], - online: usize, - target_open_round: RoundNumber, - ) { - timeout(EVENT_TIMEOUT, async { - loop { - let mut progressed = false; - for sender in 0..events.len() { - while let Ok(event) = events[sender].try_recv() { - progressed = true; - match event { - ShadowServiceEventV1::Network { recipient, message } - if sender < online && (recipient as usize) < online => - { - let recipient = recipient as usize; - match message { - NetworkMessage::RbcDagShadowCarrier(envelope) => handles - [recipient] - .carrier(sender as AuthorityIndex, envelope) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierRequest(reference) => handles - [recipient] - .carrier_request(sender as AuthorityIndex, reference) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierResponse(response) => handles - [recipient] - .carrier_response(sender as AuthorityIndex, response) - .unwrap(), - NetworkMessage::RbcDagShadowCarrierSyncRequest(request) => { - handles[recipient] - .carrier_sync_request( - sender as AuthorityIndex, - request, - ) - .unwrap(); - } - NetworkMessage::RbcDagShadowCarrierSyncResponse(response) => { - handles[recipient] - .carrier_sync_response( - sender as AuthorityIndex, - response, - ) - .unwrap(); - } - unexpected => panic!( - "autonomous shadow emitted unexpected network message: {unexpected:?}" - ), - } - } - ShadowServiceEventV1::Network { .. } => {} - ShadowServiceEventV1::ClockState { open_round, .. } => { - open_rounds[sender] = open_rounds[sender].max(open_round); - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("autonomous shadow rejected valid test traffic: {error}") - } - _ => {} - } - } - } - if open_rounds[..online] - .iter() - .all(|round| *round >= target_open_round) - { - return; - } - if !progressed { - tokio::time::sleep(Duration::from_millis(1)).await; - } else { - tokio::task::yield_now().await; - } - } - }) - .await - .unwrap_or_else(|_| panic!("online prefix did not open round {target_open_round}")); - } - async fn stop( handle: StarfishRbcDagShadowServiceHandleV1, events: mpsc::Receiver, @@ -2341,9 +1481,7 @@ mod tests { loop { match next_event(&mut events).await { ShadowServiceEventV1::Rejected { peer: None, error } => return error, - ShadowServiceEventV1::Ready { .. } => { - panic!("invalid shadow startup became ready") - } + ShadowServiceEventV1::Ready => panic!("invalid shadow startup became ready"), _ => {} } } @@ -2446,350 +1584,6 @@ mod tests { stop(handle, events, task).await; } - async fn assert_autonomous_zero_load_progress(n: usize) { - let harness = Harness::new_with_n(n); - let mut handles = Vec::new(); - let mut events = Vec::new(); - let mut tasks = Vec::new(); - for authority in 0..n as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous(authority); - loop { - match next_event(&mut node_events).await { - ShadowServiceEventV1::Ready { autonomous_clock } => { - assert!(autonomous_clock); - break; - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("autonomous shadow startup failed: {error}") - } - _ => {} - } - } - handles.push(handle); - events.push(node_events); - tasks.push(task); - } - - let mut open_rounds = vec![1; n]; - let mut deliveries = vec![0; n]; - let mut sync_requests = 0; - for (authority, handle) in handles.iter().enumerate() { - for peer in 0..n { - if peer != authority { - handle.peer_connected(peer as AuthorityIndex).unwrap(); - } - } - } - for fixed_round in 1..=6 { - for handle in &handles { - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - } - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut sync_requests, - fixed_round + 1, - ) - .await; - } - - assert!( - deliveries.iter().all(|count| *count > 0), - "every node must RBC-deliver mature heartbeat carriers: {deliveries:?}" - ); - assert!(open_rounds.iter().all(|round| *round >= 7)); - assert_eq!( - sync_requests, 0, - "healthy proactive rounds must not trigger repair polling" - ); - - drop(events); - for handle in &handles { - handle.shutdown().await.unwrap(); - } - for task in tasks { - task.await.unwrap(); - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn four_node_autonomous_zero_load_clock_delivers_mature_heartbeats() { - assert_autonomous_zero_load_progress(4).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn seven_node_autonomous_zero_load_clock_delivers_mature_heartbeats() { - assert_autonomous_zero_load_progress(7).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn autonomous_exact_slot_sync_is_bounded_and_late_response_is_idempotent() { - let harness = Harness::new(); - let heartbeat_interval = Duration::from_millis(250); - let (author, mut author_events, author_task) = - harness.start_autonomous_with_interval(0, heartbeat_interval); - let (receiver, mut receiver_events, receiver_task) = - harness.start_autonomous_with_interval(1, heartbeat_interval); - loop { - if let ShadowServiceEventV1::Ready { autonomous_clock } = - next_event(&mut author_events).await - { - assert!(autonomous_clock); - break; - } - } - loop { - if let ShadowServiceEventV1::Ready { autonomous_clock } = - next_event(&mut receiver_events).await - { - assert!(autonomous_clock); - break; - } - } - - author.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - let proactive = next_carrier(&mut author_events, 1).await; - receiver.peer_connected(0).unwrap(); - tokio::time::sleep(SHADOW_CARRIER_SYNC_MIN_GRACE_INTERVAL_V1).await; - - let request = loop { - if let ShadowServiceEventV1::Network { - recipient: 0, - message: NetworkMessage::RbcDagShadowCarrierSyncRequest(request), - } = next_event(&mut receiver_events).await - { - break request; - } - }; - assert_eq!((request.author, request.round), (0, 1)); - author.carrier_sync_request(1, request).unwrap(); - let response = loop { - if let ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrierSyncResponse(response), - } = next_event(&mut author_events).await - { - break response; - } - }; - assert_eq!(response.canonical_carrier, proactive.canonical_carrier); - assert_eq!( - response.authentication_sidecar, - proactive.authentication_sidecar - ); - receiver.carrier_sync_response(0, response.clone()).unwrap(); - loop { - match next_event(&mut receiver_events).await { - ShadowServiceEventV1::Input { - kind: "carrier_sync_response", - outcome: "authenticated", - } => break, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("valid exact-slot response was rejected: {error}") - } - _ => {} - } - } - - // A proactive/response race can leave an authenticated response in - // flight after the slot was admitted. It is an idempotent replay, not - // peer misbehavior and not a benchmark-invalidating error. - receiver.carrier_sync_response(0, response).unwrap(); - loop { - match next_event(&mut receiver_events).await { - ShadowServiceEventV1::Input { - kind: "carrier_sync_response", - outcome: "ignored_already_admitted_or_stale", - } => break, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("late exact-slot response was not idempotent: {error}") - } - _ => {} - } - } - - // One requester cannot amplify repeated reads of a retained large - // carrier faster than the bounded synchronization interval. - author.carrier_sync_request(1, request).unwrap(); - loop { - if let ShadowServiceEventV1::Input { - kind: "carrier_sync_request", - outcome: "rate_limited", - } = next_event(&mut author_events).await - { - break; - } - } - - stop(author, author_events, author_task).await; - stop(receiver, receiver_events, receiver_task).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn autonomous_exact_sync_closes_a_multi_round_gap() { - let harness = Harness::new(); - let mut handles = Vec::new(); - let mut events = Vec::new(); - let mut tasks = Vec::new(); - for authority in 0..N as AuthorityIndex { - let (handle, mut node_events, task) = harness.start_autonomous(authority); - wait_ready(&mut node_events).await; - handles.push(handle); - events.push(node_events); - tasks.push(task); - } - - // Establish round one for all validators, then let a quorum advance - // while authority 3 is offline and receives none of the proactive - // carriers. Starting the gap at round two makes reconnect request - // exact repair immediately; the one-hour normal heartbeat still - // cannot help with the later repaired rounds. - for (authority, handle) in handles.iter().enumerate() { - for peer in 0..N { - if peer != authority { - handle.peer_connected(peer as AuthorityIndex).unwrap(); - } - } - } - let mut open_rounds = vec![1; N]; - let mut deliveries = vec![0; N]; - let mut sync_requests = 0; - for handle in &handles { - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - } - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut sync_requests, - 2, - ) - .await; - for authority in 0..3 { - handles[authority].peer_disconnected(3).unwrap(); - handles[3] - .peer_disconnected(authority as AuthorityIndex) - .unwrap(); - } - for fixed_round in 2..=8 { - for handle in &handles[..3] { - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - } - pump_online_prefix_until_round( - &handles, - &mut events, - &mut open_rounds, - 3, - fixed_round + 1, - ) - .await; - } - assert_eq!(open_rounds[3], 2); - assert!(open_rounds[..3].iter().all(|round| *round >= 9)); - - // Reconnect, fix the lagging node's first local slot, and let the - // healthy quorum open one more round. Exact responses then drive an - // immediate local heartbeat per repaired round; the one-hour normal - // timer cannot be responsible for convergence. - for (authority, handle) in handles.iter().enumerate() { - for peer in 0..N { - if peer != authority { - handle.peer_connected(peer as AuthorityIndex).unwrap(); - } - } - } - handles[3] - .send(ShadowServiceMessageV1::HeartbeatTick) - .unwrap(); - for handle in &handles[..3] { - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - } - - let sync_requests_before_catch_up = sync_requests; - pump_autonomous_until_round( - &handles, - &mut events, - &mut open_rounds, - &mut deliveries, - &mut sync_requests, - 10, - ) - .await; - assert!( - sync_requests > sync_requests_before_catch_up, - "catch-up must use exact-slot repair" - ); - assert!(open_rounds.iter().all(|round| *round >= 10)); - - drop(events); - for handle in &handles { - handle.shutdown().await.unwrap(); - } - for task in tasks { - task.await.unwrap(); - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn autonomous_wal_restart_serves_the_exact_persisted_heartbeat() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start_autonomous(0); - loop { - if let ShadowServiceEventV1::Ready { autonomous_clock } = next_event(&mut events).await - { - assert!(autonomous_clock); - break; - } - } - handle.send(ShadowServiceMessageV1::HeartbeatTick).unwrap(); - let original = next_carrier(&mut events, 1).await; - stop(handle, events, task).await; - - let (restarted, mut restarted_events, restarted_task) = harness.start_autonomous(0); - let mut replayed = false; - loop { - match next_event(&mut restarted_events).await { - ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches > 0, - ShadowServiceEventV1::Ready { autonomous_clock } => { - assert!(autonomous_clock); - break; - } - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("autonomous WAL restart failed: {error}") - } - _ => {} - } - } - assert!(replayed); - restarted - .carrier_sync_request( - 1, - RbcDagShadowCarrierSyncRequest { - author: 0, - round: 1, - }, - ) - .unwrap(); - loop { - if let ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrierSyncResponse(response), - } = next_event(&mut restarted_events).await - { - assert_eq!(response.canonical_carrier, original.canonical_carrier); - assert_eq!( - response.authentication_sidecar, - original.authentication_sidecar - ); - break; - } - } - stop(restarted, restarted_events, restarted_task).await; - } - fn phase_carrier( author: AuthorityIndex, statement: RbcPhaseStatementV1, @@ -2840,7 +1634,7 @@ mod tests { loop { match next_event(&mut restarted_events).await { ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches > 0, - ShadowServiceEventV1::Ready { .. } => break, + ShadowServiceEventV1::Ready => break, ShadowServiceEventV1::Rejected { error, .. } => { panic!("valid shadow restart failed: {error}") } @@ -3117,27 +1911,6 @@ mod tests { }; assert_eq!(identity.author, 2); assert_eq!(identity.round, 1); - handle - .carrier_response( - 1, - RbcDagShadowCarrierResponse { - reference: target.reference(), - canonical_carrier: target.canonical_wire_bytes().unwrap(), - }, - ) - .unwrap(); - loop { - match next_event(&mut events).await { - ShadowServiceEventV1::Input { - kind: "recovery", - outcome: "ignored_already_retained", - } => break, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("late exact recovery response was not idempotent: {error}") - } - _ => {} - } - } handle.direct_delivered(identity).unwrap(); loop { if let ShadowServiceEventV1::Comparison(comparison) = next_event(&mut events).await { @@ -3170,11 +1943,10 @@ mod tests { #[tokio::test] async fn bounded_input_reports_overload_but_shutdown_waits_for_capacity() { - let input_capacity = shadow_input_capacity(N, ShadowServiceModeV1::DirectMirror).unwrap(); + let input_capacity = shadow_input_capacity(N).unwrap(); let (sender, mut receiver) = mpsc::channel(input_capacity); let handle = StarfishRbcDagShadowServiceHandleV1 { sender, - mode: ShadowServiceModeV1::DirectMirror, max_sidecar_size: 3 + N * MAC_TAG_SIZE, own_authority: 0, committee_size: N, @@ -3224,7 +1996,6 @@ mod tests { let (sender, _receiver) = mpsc::channel(1); let oversized = StarfishRbcDagShadowServiceHandleV1 { sender, - mode: ShadowServiceModeV1::DirectMirror, max_sidecar_size: 3 + N * MAC_TAG_SIZE, own_authority: 0, committee_size: N, @@ -3251,14 +2022,12 @@ mod tests { #[tokio::test] async fn sixty_validator_burst_fits_before_the_actor_drains() { const LARGE_N: usize = 60; - let input_capacity = - shadow_input_capacity(LARGE_N, ShadowServiceModeV1::DirectMirror).unwrap(); + let input_capacity = shadow_input_capacity(LARGE_N).unwrap(); assert_eq!(input_capacity, SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1); let (sender, _receiver) = mpsc::channel(input_capacity); let invalidated = Arc::new(Mutex::new(None)); let handle = StarfishRbcDagShadowServiceHandleV1 { sender, - mode: ShadowServiceModeV1::DirectMirror, max_sidecar_size: 3 + LARGE_N * MAC_TAG_SIZE, own_authority: 0, committee_size: LARGE_N, @@ -3283,27 +2052,11 @@ mod tests { } assert_eq!(*invalidated.lock(), None); assert!(matches!( - shadow_input_capacity(LARGE_N + 1, ShadowServiceModeV1::DirectMirror), + shadow_input_capacity(LARGE_N + 1), Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { .. }) )); } - #[test] - fn autonomous_burst_budget_accepts_twenty_and_rejects_twenty_one() { - let mode = ShadowServiceModeV1::AutonomousClock { - heartbeat_interval: Duration::from_millis(250), - }; - assert_eq!(shadow_input_capacity(20, mode).unwrap(), 64); - assert!(matches!( - shadow_input_capacity(21, mode), - Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { - committee_size: 21, - required_capacity: 65, - maximum_capacity: 64, - }) - )); - } - #[tokio::test] async fn dropping_all_handles_stops_actor_despite_retry_timer() { let harness = Harness::new(); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 11868ad7..7564e7c5 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -52,13 +52,6 @@ impl Validator { ) .map_err(|error| eyre!(error))?; let is_starfish_rbc = protocol_config.consensus_protocol.is_starfish_rbc(); - if public_config.parameters.starfish_rbc_dag_autonomous_clock - && !public_config.parameters.starfish_rbc_dag_shadow - { - return Err(eyre!( - "Starfish-RBC-DAG autonomous clock requires the RBC-DAG shadow" - )); - } if public_config .parameters .starfish_rbc_single_dag_echo_qc_fast_path @@ -81,11 +74,6 @@ impl Validator { benchmarks" )); } - if public_config.parameters.starfish_rbc_dag_autonomous_clock && !is_starfish_rbc { - return Err(eyre!( - "Starfish-RBC-DAG autonomous clock requires consensus 'starfish-rbc'" - )); - } if public_config.parameters.starfish_rbc_dag_shadow && !is_starfish_rbc { return Err(eyre!( "Starfish-RBC-DAG shadow mode requires consensus 'starfish-rbc'" @@ -234,12 +222,7 @@ impl Validator { } else { None }; - let starfish_rbc_dag_shadow_wal = - if public_config.parameters.starfish_rbc_dag_autonomous_clock { - private_config.starfish_rbc_dag_autonomous_clock_wal() - } else { - private_config.starfish_rbc_dag_shadow_wal() - }; + let starfish_rbc_dag_shadow_wal = private_config.starfish_rbc_dag_shadow_wal(); let (core, bls_cert_aggregator) = Core::open( block_handler, @@ -393,101 +376,6 @@ mod smoke_tests { })); } - #[tokio::test] - async fn autonomous_clock_requires_shadow_mode() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_autonomous_clock = true; - public_config - .parameters - .refresh_starfish_rbc_protocol_instance(); - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish-rbc".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("autonomous clock requires the RBC-DAG shadow") - })); - } - - #[tokio::test] - async fn autonomous_clock_rejects_non_rbc_protocol() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_shadow = true; - public_config.parameters.starfish_rbc_dag_autonomous_clock = true; - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("autonomous clock requires consensus 'starfish-rbc'") - })); - } - - #[tokio::test] - async fn autonomous_clock_rejects_zero_heartbeat_interval() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_shadow = true; - public_config.parameters.starfish_rbc_dag_autonomous_clock = true; - public_config - .parameters - .starfish_rbc_dag_heartbeat_interval_ms = 0; - public_config - .parameters - .refresh_starfish_rbc_protocol_instance(); - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish-rbc".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("autonomous heartbeat interval must be greater than zero") - })); - } - async fn run_commit_test( consensus: &str, block_authentication: Option<&str>, @@ -501,23 +389,6 @@ mod smoke_tests { block_authentication: Option<&str>, port_offset: u16, starfish_rbc_dag_shadow: bool, - ) { - run_commit_test_with_shadow_mode( - consensus, - block_authentication, - port_offset, - starfish_rbc_dag_shadow, - false, - ) - .await; - } - - async fn run_commit_test_with_shadow_mode( - consensus: &str, - block_authentication: Option<&str>, - port_offset: u16, - starfish_rbc_dag_shadow: bool, - autonomous_clock: bool, ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); @@ -525,12 +396,6 @@ mod smoke_tests { NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); public_config.parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; - public_config.parameters.starfish_rbc_dag_autonomous_clock = autonomous_clock; - if autonomous_clock { - public_config - .parameters - .starfish_rbc_dag_heartbeat_interval_ms = 50; - } if consensus == "starfish-rbc" { public_config .parameters @@ -578,48 +443,7 @@ mod smoke_tests { ), } - if autonomous_clock { - tokio::time::timeout(timeout, async { - loop { - if validators.iter().all(|validator| { - let metrics = validator.metrics(); - metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1 - && metrics.starfish_rbc_dag_shadow_carrier_round.get() > 3 - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["heartbeat", "accepted"]) - .get() - > 0 - && metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .get() - > 0 - && metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get() - > 0 - && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 - }) { - break; - } - time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .expect("autonomous carrier clock did not advance while direct RBC committed"); - - for validator in &validators { - let metrics = validator.metrics(); - assert_eq!(metrics.starfish_rbc_dag_shadow_clock_valid.get(), 1); - assert!(metrics.starfish_rbc_dag_shadow_carrier_round.get() > 3); - assert_eq!( - metrics.starfish_rbc_dag_shadow_comparison_valid.get(), - 0, - "autonomous mode must not claim direct-round comparison" - ); - } - } else if starfish_rbc_dag_shadow { + if starfish_rbc_dag_shadow { let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR * i64::try_from(committee_size).unwrap(); tokio::time::timeout(timeout, async { @@ -761,11 +585,6 @@ mod smoke_tests { run_commit_test_with_shadow("starfish-rbc", Some("mac"), 1640, true).await; } - #[tokio::test] - async fn starfish_rbc_dag_autonomous_clock_advances_without_owning_consensus() { - run_commit_test_with_shadow_mode("starfish-rbc", Some("mac"), 1700, true, true).await; - } - #[tokio::test] async fn starfish_rbc_single_validator_starts_on_current_thread_runtime() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 39344955..6a448240 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -76,13 +76,6 @@ enum Operation { /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_heartbeat_interval_ms: Option, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -120,13 +113,6 @@ enum Operation { /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_autonomous_clock: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_heartbeat_interval_ms: Option, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -186,19 +172,12 @@ enum Operation { /// `starfish-rbc`. #[clap(long, default_value_t = false)] starfish_rbc_dag_shadow: bool, - /// Let the non-authoritative Starfish-RBC-DAG shadow create its own - /// optimistic carrier clock. Requires `--starfish-rbc-dag-shadow`. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_autonomous_clock: bool, /// Testbed-only: deliver a single-DAG RBC header after a receiver-local /// quorum ECHO. This preserves uniqueness but not Byzantine /// selective-withholding totality, so it is restricted to finite /// benchmark runs. #[clap(long, default_value_t = false)] starfish_rbc_single_dag_echo_qc_fast_path: bool, - /// Maximum interval between autonomous RBC-DAG heartbeat carriers. - #[clap(long, value_name = "INT")] - starfish_rbc_dag_heartbeat_interval_ms: Option, #[clap(long, value_name = "INT", default_value_t = 600)] duration_secs: u64, /// Dissemination mode override: @@ -234,8 +213,6 @@ async fn main() -> Result<()> { consensus: consensus_protocol, block_authentication, starfish_rbc_dag_shadow, - starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, } => { run( authority, @@ -247,8 +224,6 @@ async fn main() -> Result<()> { consensus_protocol, block_authentication, starfish_rbc_dag_shadow, - starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, ) .await? } @@ -264,8 +239,6 @@ async fn main() -> Result<()> { consensus: consensus_protocol, block_authentication, starfish_rbc_dag_shadow, - starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, data_dir, base_ip, storage_backend, @@ -286,8 +259,6 @@ async fn main() -> Result<()> { consensus_protocol, block_authentication, starfish_rbc_dag_shadow, - starfish_rbc_dag_autonomous_clock, - starfish_rbc_dag_heartbeat_interval_ms, data_dir, base_ip, storage_backend, @@ -310,9 +281,7 @@ async fn main() -> Result<()> { consensus: consensus_protocol, block_authentication, starfish_rbc_dag_shadow, - starfish_rbc_dag_autonomous_clock, starfish_rbc_single_dag_echo_qc_fast_path, - starfish_rbc_dag_heartbeat_interval_ms, duration_secs, dissemination_mode, } => { @@ -324,12 +293,8 @@ async fn main() -> Result<()> { node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; - node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = starfish_rbc_single_dag_echo_qc_fast_path; - if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; - } if is_starfish_rbc_selection(&consensus_protocol) { node_parameters.refresh_starfish_rbc_protocol_instance(); } @@ -480,8 +445,6 @@ async fn local_benchmark( }; let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; - let starfish_rbc_dag_autonomous_clock_expected = - node_parameters.starfish_rbc_dag_autonomous_clock; // Create temporary directories for each validator let base_dir = PathBuf::from("local-benchmark"); @@ -554,22 +517,14 @@ async fn local_benchmark( ) .await? }; - let validator_metrics = validator.metrics(); if !is_byzantine { - metrics_of_honest_validators.push(Arc::clone(&validator_metrics)); + metrics_of_honest_validators.push(validator.metrics()); reporters_of_honest_validators.push(validator.reporter()) } // Use the same pattern as the run method let handle = tokio::spawn(async move { let (network_result, _metrics_result) = validator.await_completion().await; - if starfish_rbc_dag_autonomous_clock_expected { - validator_metrics.starfish_rbc_dag_shadow_clock_valid.set(0); - } else if starfish_rbc_dag_shadow_expected { - validator_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } network_result }); abort_handles.push(handle.abort_handle()); @@ -579,13 +534,10 @@ async fn local_benchmark( if starfish_rbc_dag_shadow_expected { let ready = tokio::time::timeout(Duration::from_secs(30), async { loop { - if metrics_of_honest_validators.iter().all(|metrics| { - if starfish_rbc_dag_autonomous_clock_expected { - metrics.starfish_rbc_dag_shadow_clock_valid.get() == 1 - } else { - metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1 - } - }) { + if metrics_of_honest_validators + .iter() + .all(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) + { break; } tokio::time::sleep(Duration::from_millis(25)).await; @@ -597,24 +549,12 @@ async fn local_benchmark( abort_handle.abort(); } fs::remove_dir_all(&base_dir)?; - let mode = if starfish_rbc_dag_autonomous_clock_expected { - "autonomous clock" - } else { - "direct-comparison shadow" - }; eyre::bail!( - "Starfish-RBC-DAG {mode} did not become ready on every honest validator; benchmark was not started" + "Starfish-RBC-DAG shadow did not become ready on every honest validator; benchmark was not started" ); } } - let autonomous_clock_baselines = starfish_rbc_dag_autonomous_clock_expected.then(|| { - metrics_of_honest_validators - .iter() - .map(|metrics| metrics.autonomous_clock_benchmark_baseline()) - .collect::>() - }); - // Run for specified duration tokio::select! { _ = tokio::time::sleep(Duration::from_secs(duration_secs)) => { @@ -629,8 +569,6 @@ async fn local_benchmark( duration_secs, committee_size, starfish_rbc_dag_shadow_expected, - starfish_rbc_dag_autonomous_clock_expected, - autonomous_clock_baselines.clone(), ); // Abort all tasks @@ -656,11 +594,9 @@ async fn local_benchmark( duration_secs, committee_size, starfish_rbc_dag_shadow_expected, - starfish_rbc_dag_autonomous_clock_expected, - autonomous_clock_baselines, ); fs::remove_dir_all(base_dir)?; - eyre::bail!("All validators completed before the requested benchmark duration") + Ok(()) } } } @@ -676,8 +612,6 @@ async fn run( consensus_protocol: String, block_authentication: Option, starfish_rbc_dag_shadow: bool, - starfish_rbc_dag_autonomous_clock: bool, - starfish_rbc_dag_heartbeat_interval_ms: Option, ) -> Result<()> { tracing::info!("Starting node {authority}"); @@ -692,14 +626,6 @@ async fn run( if starfish_rbc_dag_shadow { public_config.parameters.starfish_rbc_dag_shadow = true; } - if starfish_rbc_dag_autonomous_clock { - public_config.parameters.starfish_rbc_dag_autonomous_clock = true; - } - if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { - public_config - .parameters - .starfish_rbc_dag_heartbeat_interval_ms = interval_ms; - } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" ))?; @@ -738,8 +664,6 @@ async fn dryrun( consensus_protocol: String, block_authentication: Option, starfish_rbc_dag_shadow: bool, - starfish_rbc_dag_autonomous_clock: bool, - starfish_rbc_dag_heartbeat_interval_ms: Option, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -783,10 +707,6 @@ async fn dryrun( node_parameters.compress_network = compress_network; node_parameters.block_authentication = block_authentication; node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; - node_parameters.starfish_rbc_dag_autonomous_clock = starfish_rbc_dag_autonomous_clock; - if let Some(interval_ms) = starfish_rbc_dag_heartbeat_interval_ms { - node_parameters.starfish_rbc_dag_heartbeat_interval_ms = interval_ms; - } ensure_starfish_rbc_protocol_instance(&consensus_protocol, &mut node_parameters); if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; @@ -1025,10 +945,7 @@ mod tests { "--block-authentication", "mac", "--starfish-rbc-dag-shadow", - "--starfish-rbc-dag-autonomous-clock", "--starfish-rbc-single-dag-echo-qc-fast-path", - "--starfish-rbc-dag-heartbeat-interval-ms", - "125", ]) .unwrap(); @@ -1036,9 +953,7 @@ mod tests { consensus, block_authentication, starfish_rbc_dag_shadow, - starfish_rbc_dag_autonomous_clock, starfish_rbc_single_dag_echo_qc_fast_path, - starfish_rbc_dag_heartbeat_interval_ms, .. } = args.operation else { @@ -1047,17 +962,13 @@ mod tests { assert_eq!(consensus, "starfish-rbc"); assert_eq!(block_authentication.as_deref(), Some("mac")); assert!(starfish_rbc_dag_shadow); - assert!(starfish_rbc_dag_autonomous_clock); assert!(starfish_rbc_single_dag_echo_qc_fast_path); - assert_eq!(starfish_rbc_dag_heartbeat_interval_ms, Some(125)); } #[test] fn dry_run_starfish_rbc_configuration_gets_a_protocol_instance() { let mut parameters = NodeParameters { starfish_rbc_dag_shadow: true, - starfish_rbc_dag_autonomous_clock: true, - starfish_rbc_dag_heartbeat_interval_ms: 125, ..NodeParameters::default() }; @@ -1069,7 +980,5 @@ mod tests { .is_some_and(|instance| instance != [0; 32]) ); assert!(parameters.starfish_rbc_dag_shadow); - assert!(parameters.starfish_rbc_dag_autonomous_clock); - assert_eq!(parameters.starfish_rbc_dag_heartbeat_interval_ms, 125); } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index ae5da439..9753358d 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -11,12 +11,10 @@ committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoi full validator recovery remain incomplete The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not -implemented. The milestone-three direct-header comparison runtime is enabled with `--consensus -starfish-rbc --starfish-rbc-dag-shadow`. Milestone four adds a separate control-only runtime with -`--starfish-rbc-dag-autonomous-clock`; its carrier rounds advance independently through -authenticated admission and empty heartbeats. Both modes leave the direct prototype's DAG, -pacemaker, commit, and output unchanged. The eventual protocol is new, not a transport option or a -version-two alias for `starfish-rbc`. +implemented. The current runtime is enabled with `--consensus starfish-rbc +--starfish-rbc-dag-shadow`; it observes the direct prototype without changing its DAG, pacemaker, +commit, or output. The eventual protocol is new, not a transport option or a version-two alias for +`starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -49,23 +47,13 @@ This is a proposed composition. The reliable-broadcast thresholds are standard, commit rules already exist. Milestone two provides a canonical codec plus deterministic carrier/RBC, certified-projection, decision, and crash-journal models. Milestone three adds an opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired -direct/shadow delivery observations. Milestone four adds an independently authenticated autonomous -heartbeat namespace, sequential `Q`-admitted carrier clock, bounded future buffering, exact-slot -synchronization, and clock-state metrics. The direct `starfish-rbc` service remains the only -authority: shadow admission, delivery, recovery, clock advancement, or failure cannot advance a -proposal, mark a DAG vertex clean, vote, commit, or order output. - -Autonomous mode is intentionally control-only at this milestone. It ignores direct application -headers and uses a distinct WAL and authentication protocol instance. This avoids falsely equating -direct consensus rounds with faster carrier rounds, but also means milestone four does not yet -measure application latency through the new carrier DAG. Application-origin assignment and the -certified consensus projection remain later milestones. - -Shadow restart coverage is deliberately scoped to reopening the actor and its WAL: mirror mode -requires an identical recovered direct-header history, while autonomous mode reopens its -control-only heartbeat history without direct headers and serves byte-identical exact-slot -responses. This is not a full validator crash-recovery claim. The authoritative direct -`starfish-rbc` baseline does not yet durably record its remote-slot +direct/shadow delivery observations. The direct `starfish-rbc` service remains the only authority: +shadow admission, delivery, recovery, or failure cannot advance a proposal, mark a DAG vertex +clean, vote, commit, or order output. + +Milestone-three restart coverage is deliberately scoped to reopening the shadow actor and its WAL +against an identical recovered direct-header history. It is not a full validator crash-recovery +claim. The authoritative direct `starfish-rbc` baseline does not yet durably record its remote-slot ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest recovered own header dirty. Full validator restart remains fail-stop until those direct-RBC locks @@ -78,9 +66,7 @@ can perturb authoritative timing even though no shadow result is consumed by con one has no feature handshake. Every validator in a shadow run must use a binary that understands the append-only shadow wire variants and the flag must be deployed committee-wide; an older peer will reject an unknown bincode variant and may close the shared connection. Mixed-version or -partially enabled runs are not valid comparisons. Mirror and autonomous carriers derive distinct -authentication protocol instances so they cannot cross-admit, but this fail-closed boundary is not -a substitute for capability negotiation. +partially enabled runs are not valid comparisons. Shadow shutdown is bounded so an observational WAL failure cannot indefinitely block validator shutdown. If that timeout fires, the blocking worker may still hold the shadow WAL's single-writer @@ -785,22 +771,11 @@ newest current-process observation (`<= 4`). These are empirical benchmark cover asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as proof of a protocol failure. -The actor reserves the full hard 64-entry queue so several fan-in bursts can wait behind a -synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars and -allocator overhead). Mirror mode budgets one peer fan-in plus five local/control inputs and accepts -at most 60 validators. Autonomous mode budgets a simultaneous carrier, exact-slot request, and -exact-slot response per peer plus five control inputs and accepts at most 20 validators. Larger runs -are rejected rather than silently producing incomplete evidence. Timer notifications are coalesced, -healthy proactive rounds receive a repair grace period, and exact synchronization is rate-limited -per peer. Requested historical slots remain recoverable beyond the benchmark-only unsolicited -retention window. - -Autonomous benchmark validity is separate from delivery comparison validity. -`starfish_rbc_dag_shadow_clock_valid` must remain `1`, the WAL and heartbeat counters must progress, -the carrier round and embedded-RBC delivery count must advance during the measured interval, -recovery must drain, and the reported clock-state/backlog and cross-node skew must remain within the -configured empirical guards. These checks establish that the observational carrier plane stayed -live and bounded; they are not a partial-synchrony proof. +The milestone-three actor reserves the full hard 64-entry queue so several fan-in bursts can wait +behind a synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars +and allocator overhead). It still verifies that one peer fan-in plus five local/control inputs fits; +shadow runs above 60 validators are rejected rather than silently producing an incomplete +comparison. ## 15. Safety obligations @@ -873,12 +848,7 @@ minimum it must cover: - persisted shadow-actor restart with byte-identical retransmission against an identical recovered direct-header history, bounded overload, poisoned-tag candidate retention, exact recovery, and paired delivery observations against the current direct RBC kernel. Full validator restart is - excluded until the authoritative direct-RBC locks are durable; and -- autonomous actor progress at `n = 4` and `n = 7`, no steady-state repair polling on healthy - proactive rounds, exact-slot synchronization with idempotent late responses and per-peer rate - limiting, multi-round convergence after a validator falls behind, control-only WAL reopen, - distinct authentication namespace, and an integration check that direct Starfish-RBC continues - committing while the observational carrier clock advances. + excluded until the authoritative direct-RBC locks are durable. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -908,15 +878,6 @@ transition and validates through a clone-based reference reducer. Those costs ar charged as protocol overhead: performance runs require incremental state transitions/checkpoints or an equivalently durable baseline, plus separate WAL/fsync accounting. -As an implementation-continuity check, a 10-validator, 60-second local run on 2026-08-11 used the -AWS RTT emulator, a nominal 1,000 tx/s load, MAC authentication, and a 250 ms autonomous heartbeat. -All validators reached carrier round 196 with zero skew and zero pending recovery; the control -plane recorded 1,959 heartbeats and 19,280 embedded-RBC deliveries. The authoritative direct -Starfish-RBC path reported 776.50 tx/s, 3,378.4 ms p50 block latency, 3,953.8 ms p50 end-to-end -latency, and 0.45 MB/s average outbound bandwidth. This is not a comparative performance claim: -the cutoff includes the local generator warmup, and the control-only shadow still performs -per-transition fsync/reference-model work on the authoritative network socket. - ## 19. Contained implementation milestones Every milestone is committed separately. @@ -928,17 +889,15 @@ Every milestone is committed separately. frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable journal models; and deterministic adversarial simulations. No network or existing consensus path changes. -3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the - current direct `starfish-rbc` service, cache the validated committee/domain identity rather than - re-hashing all public keys per carrier, journal ingress and local locks, and compare embedded - versus direct RBC delivery through current-process paired observations. Direct RBC remains - authoritative; shadow results never affect proposals or commits. The reference WAL/reducer is a - correctness instrument, not yet an interpretable protocol-performance path. -4. **Optimistic carrier clock (implemented, opt-in control shadow):** run a separately namespaced, - control-only heartbeat carrier plane with the distinct authenticated-admission latch, sequential - quorum clock, bounded future buffer, exact-slot synchronization, durable restart, and clock - validity metrics while consensus still uses the current direct baseline. Application headers are - not assigned to autonomous carrier rounds yet. +3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the current direct + `starfish-rbc` service, cache the validated committee/domain identity rather than re-hashing all + public keys per carrier, journal ingress and local locks, and compare embedded versus direct RBC + delivery through current-process paired observations. Direct RBC remains authoritative; shadow + results never affect proposals or commits. The reference WAL/reducer is a correctness instrument, + not yet an interpretable protocol-performance path. +4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum + clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses + the current baseline. 5. **Authoritative embedded RBC:** remove direct ECHO/READY authority only after shadow tests show identical delivery under reordering, loss, equivocation, poisoned tags, and restart. 6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit @@ -957,8 +916,7 @@ the executable model or measured prototype: - production maximum future-carrier buffer and payload runahead (the executable model deliberately uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); -- the production control-heartbeat rate under low load and backpressure (the autonomous shadow's - configurable 250 ms default is an empirical test value, not a protocol constant); +- the control-heartbeat rate under low load and backpressure; - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; - quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and From a5358cd750aa7269825665c450b5b18f2a4af8dd Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:46:06 +0200 Subject: [PATCH 59/62] Revert "Add persisted Starfish-RBC-DAG shadow runtime" This reverts commit 445736ba7ba20176ed08667526113e47969510ee. --- README.md | 27 +- crates/orchestrator/src/benchmark.rs | 48 +- crates/orchestrator/src/main.rs | 16 - crates/orchestrator/src/measurements.rs | 670 ----- crates/orchestrator/src/orchestrator.rs | 140 +- crates/orchestrator/src/protocol/starfish.rs | 102 +- crates/starfish-core/src/config.rs | 12 - .../starfish-core/src/core_thread/spawned.rs | 11 +- crates/starfish-core/src/lib.rs | 2 - crates/starfish-core/src/metrics.rs | 300 -- crates/starfish-core/src/net_sync.rs | 427 --- crates/starfish-core/src/network.rs | 194 +- .../src/starfish_rbc_dag/journal.rs | 176 +- .../starfish-core/src/starfish_rbc_dag/mod.rs | 797 +---- .../src/starfish_rbc_dag/model.rs | 795 +---- .../src/starfish_rbc_dag/projection.rs | 28 +- .../src/starfish_rbc_dag/storage.rs | 1388 --------- .../src/starfish_rbc_dag_shadow.rs | 2659 ----------------- .../src/starfish_rbc_dag_shadow_service.rs | 2071 ------------- crates/starfish-core/src/syncer.rs | 28 +- crates/starfish-core/src/validator.rs | 142 - crates/starfish/src/main.rs | 61 +- docs/starfish-rbc-dag-protocol.md | 103 +- 23 files changed, 160 insertions(+), 10037 deletions(-) delete mode 100644 crates/starfish-core/src/starfish_rbc_dag/storage.rs delete mode 100644 crates/starfish-core/src/starfish_rbc_dag_shadow.rs delete mode 100644 crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs diff --git a/README.md b/README.md index 7445b861..3408e20c 100644 --- a/README.md +++ b/README.md @@ -48,27 +48,12 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is a follow-up that pipelines all-carrier RBC through an optimistic carrier DAG -while keeping certified Starfish consensus and ordering in a separate logical projection. Its -canonical types, deterministic models, crash journal, and an opt-in persisted network shadow are -implemented. Run the shadow with `--consensus starfish-rbc --starfish-rbc-dag-shadow`; direct -Starfish-RBC remains solely authoritative and shadow failures or results cannot affect proposals, -commits, or output as protocol state. Shadow traffic still shares the validator's network socket and -bandwidth, so it can perturb timing, and it must be enabled only on a homogeneous new-binary -committee; there is no rolling-upgrade capability negotiation. The provisional `starfish-rbc-dag` -selector is not implemented yet. The shadow uses per-transition fsync and a clone-based reference -reducer, so it is a correctness instrument, not a fair performance baseline, and carries no safety -or liveness claim. Its WAL can reopen the shadow actor against matching recovered direct headers, -but this is not full validator crash recovery: authoritative direct Starfish-RBC phase and delivery -locks are not durable yet, so that baseline remains fail-stop across process restart. The design and -proof obligations are documented in the -[protocol design](docs/starfish-rbc-dag-protocol.md). -For any shadow comparison, -`starfish_rbc_dag_shadow_comparison_valid` must stay at `1`; a value of `0` means the bounded -observational path was disabled or shed work and the comparison must be discarded. Healthy live -production retains a short embedded-RBC pipeline tail, so benchmark validation uses bounded -unpaired-count and oldest-round-lag gauges rather than requiring instantaneous equality between -the cumulative direct and shadow delivery counters. +**Starfish-RBC-DAG** is a codec-and-model-only follow-up that pipelines all-carrier RBC through an +optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate +logical projection. Its canonical types and deterministic executable models are implemented in +isolation, but there is no network/runtime path and no safety or liveness claim. Its provisional CLI +name is `starfish-rbc-dag`, but that selector is not implemented yet. The design, current boundary, +and proof obligations are documented in the [protocol design](docs/starfish-rbc-dag-protocol.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index dc024546..cec244f1 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -99,32 +99,6 @@ pub struct BenchmarkRunSummary { pub ready_nodes_at_boot: usize, #[serde(default)] pub metrics_contributors: usize, - #[serde(default)] - pub shadow_comparison_enabled: bool, - #[serde(default)] - pub shadow_comparison_valid: bool, - #[serde(default)] - pub shadow_comparison_valid_nodes: usize, - #[serde(default)] - pub shadow_direct_deliveries: usize, - #[serde(default)] - pub shadow_deliveries: usize, - #[serde(default)] - pub shadow_delivery_matches: usize, - #[serde(default)] - pub shadow_delivery_mismatches: usize, - #[serde(default)] - pub shadow_delivery_ambiguous: usize, - #[serde(default)] - pub shadow_wal_durable_records: usize, - #[serde(default)] - pub shadow_pending_recovery: usize, - #[serde(default)] - pub shadow_unpaired_direct: usize, - #[serde(default)] - pub shadow_unpaired_shadow: usize, - #[serde(default)] - pub shadow_unpaired_max_round_lag: usize, } impl BenchmarkRunSummary { @@ -143,14 +117,7 @@ impl BenchmarkRunSummary { db_size_per_round_p25_bytes,db_size_per_round_p50_bytes,\ db_size_per_round_p75_bytes,\ block_sync_requests_sent_per_round_avg,block_header_size_avg_bytes,\ - ready_nodes_at_boot,metrics_contributors,\ - shadow_comparison_enabled,shadow_comparison_valid,\ - shadow_comparison_valid_nodes,shadow_direct_deliveries,shadow_deliveries,\ - shadow_delivery_matches,\ - shadow_delivery_mismatches,shadow_delivery_ambiguous,\ - shadow_wal_durable_records,shadow_pending_recovery,\ - shadow_unpaired_direct,shadow_unpaired_shadow,\ - shadow_unpaired_max_round_lag" + ready_nodes_at_boot,metrics_contributors" } pub fn csv_record(&self) -> String { @@ -184,19 +151,6 @@ impl BenchmarkRunSummary { format!("{:.3}", self.block_header_size_avg_bytes), self.ready_nodes_at_boot.to_string(), self.metrics_contributors.to_string(), - self.shadow_comparison_enabled.to_string(), - self.shadow_comparison_valid.to_string(), - self.shadow_comparison_valid_nodes.to_string(), - self.shadow_direct_deliveries.to_string(), - self.shadow_deliveries.to_string(), - self.shadow_delivery_matches.to_string(), - self.shadow_delivery_mismatches.to_string(), - self.shadow_delivery_ambiguous.to_string(), - self.shadow_wal_durable_records.to_string(), - self.shadow_pending_recovery.to_string(), - self.shadow_unpaired_direct.to_string(), - self.shadow_unpaired_shadow.to_string(), - self.shadow_unpaired_max_round_lag.to_string(), ] .join(",") } diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index eff22b62..1e39dcc9 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -63,10 +63,6 @@ pub struct Opts { #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac", global = true)] block_authentication: Option, - /// Run the embedded Starfish-RBC-DAG implementation as a non-authoritative shadow. - #[clap(long, global = true)] - starfish_rbc_dag_shadow: bool, - /// The type of operation to run. #[clap(subcommand)] operation: Operation, @@ -856,7 +852,6 @@ fn load_benchmark_configs( compress_network: Option, bls_workers: Option, block_authentication: &Option, - starfish_rbc_dag_shadow: bool, ) -> eyre::Result<(NodeParameters, ClientParameters)> { let mut node_parameters = match &settings.node_parameters_path { Some(path) => NodeParameters::load(path).wrap_err("Failed to load node's parameters")?, @@ -867,9 +862,6 @@ fn load_benchmark_configs( if block_authentication.is_some() { node_parameters.block_authentication = block_authentication.clone(); } - if starfish_rbc_dag_shadow { - node_parameters.starfish_rbc_dag_shadow = true; - } if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -1050,7 +1042,6 @@ async fn run( .wrap_err("Failed to crate testbed")?; let block_authentication = opts.block_authentication.clone(); - let starfish_rbc_dag_shadow = opts.starfish_rbc_dag_shadow; match opts.operation { Operation::Testbed { action } => match action { // Display the current status of the testbed. @@ -1248,7 +1239,6 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, )?; display::newline(); @@ -1416,7 +1406,6 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, )?; display::newline(); @@ -1624,7 +1613,6 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, )?; display::newline(); @@ -1791,7 +1779,6 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, )?; display::newline(); @@ -1998,7 +1985,6 @@ async fn run( compress_network, resolved_bls_workers.override_workers, &block_authentication, - starfish_rbc_dag_shadow, )?; display::newline(); @@ -2344,14 +2330,12 @@ mod tests { "benchmark", "--block-authentication", "mac", - "--starfish-rbc-dag-shadow", "--protocols", "starfish-rbc", ]) .unwrap(); assert_eq!(opts.block_authentication.as_deref(), Some("mac")); - assert!(opts.starfish_rbc_dag_shadow); let Operation::Benchmark { protocols, .. } = opts.operation else { panic!("expected benchmark operation"); }; diff --git a/crates/orchestrator/src/measurements.rs b/crates/orchestrator/src/measurements.rs index d4a78d4d..017ee561 100644 --- a/crates/orchestrator/src/measurements.rs +++ b/crates/orchestrator/src/measurements.rs @@ -15,9 +15,6 @@ use itertools::Itertools; use prettytable::{Table, row}; use prometheus_parse::Scrape; use serde::{Deserialize, Serialize}; -use starfish_core::metrics::{ - STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, -}; use crate::{ benchmark::{BenchmarkParameters, BenchmarkRunSummary, PercentileSummary}, @@ -229,68 +226,6 @@ impl Measurement { } _ => panic!("Unexpected scraped value: '{x}'"), }, - x if matches!( - x.as_str(), - "starfish_rbc_dag_shadow_inputs_total" - | "starfish_rbc_dag_shadow_delivery_comparisons_total" - ) => - { - match sample.value { - prometheus_parse::Value::Counter(value) => { - let shadow_bucket = if x - == "starfish_rbc_dag_shadow_delivery_comparisons_total" - { - sample - .labels - .get("outcome") - .map(str::to_owned) - .unwrap_or(label) - } else { - match (sample.labels.get("kind"), sample.labels.get("outcome")) { - (Some(kind), Some(outcome)) => format!("{kind},{outcome}"), - _ => label, - } - }; - measurement - .count_buckets - .insert(shadow_bucket, value as usize); - measurement.count = measurement.count_buckets.values().sum(); - } - _ => panic!("Unexpected scraped value: '{x}'"), - } - } - x if matches!( - x.as_str(), - "starfish_rbc_dag_shadow_wal_durable_batches_total" - | "starfish_rbc_dag_shadow_wal_durable_records_total" - | "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total" - ) => - { - match sample.value { - prometheus_parse::Value::Counter(value) => { - measurement.count = value as usize; - measurement.scalar = value; - } - _ => panic!("Unexpected scraped value: '{x}'"), - } - } - x if matches!( - x.as_str(), - "starfish_rbc_dag_shadow_wal_replayed_batches" - | "starfish_rbc_dag_shadow_pending_recovery" - | "starfish_rbc_dag_shadow_comparison_valid" - | "starfish_rbc_dag_shadow_unpaired_direct" - | "starfish_rbc_dag_shadow_unpaired_shadow" - | "starfish_rbc_dag_shadow_unpaired_max_round_lag" - ) => - { - match sample.value { - prometheus_parse::Value::Gauge(value) => { - measurement.scalar = value; - } - _ => panic!("Unexpected scraped value: '{x}'"), - } - } _ => { measurements.remove(&sample.metric); } @@ -358,10 +293,6 @@ pub struct MeasurementsCollection { /// Validators that contributed at least one parsed measurement sample. #[serde(default)] observed_scrapers: BTreeSet, - /// Validators represented only by a synthetic missing-final-scrape - /// invalidation marker, not by a successfully parsed metrics response. - #[serde(default)] - synthetic_only_scrapers: BTreeSet, } impl MeasurementsCollection { @@ -377,7 +308,6 @@ impl MeasurementsCollection { db_sizes: Vec::new(), ready_nodes_at_boot, observed_scrapers: BTreeSet::new(), - synthetic_only_scrapers: BTreeSet::new(), } } @@ -391,7 +321,6 @@ impl MeasurementsCollection { /// Add a new measurement to the collection. pub fn add(&mut self, scraper_id: ScraperId, label: String, measurement: Measurement) { self.observed_scrapers.insert(scraper_id); - self.synthetic_only_scrapers.remove(&scraper_id); self.data .entry(label) .or_default() @@ -408,33 +337,6 @@ impl MeasurementsCollection { self.ready_nodes_at_boot = ready_nodes_at_boot.min(self.parameters.nodes); } - /// Record that an initially-live validator did not provide a usable final - /// shadow scrape. Appending an explicit invalid observation prevents an - /// earlier successful scrape from being mistaken for fresh final evidence. - pub fn mark_shadow_final_scrape_missing(&mut self, scraper_id: ScraperId) { - let timestamp = self - .data - .values() - .filter_map(|by_scraper| by_scraper.get(&scraper_id)) - .filter_map(|series| series.last()) - .map(Measurement::timestamp) - .max() - .unwrap_or_default(); - if !self.observed_scrapers.contains(&scraper_id) { - self.synthetic_only_scrapers.insert(scraper_id); - } - self.data - .entry("starfish_rbc_dag_shadow_comparison_valid".to_owned()) - .or_default() - .entry(scraper_id) - .or_default() - .push(Measurement { - timestamp, - scalar: 0.0, - ..Measurement::default() - }); - } - /// Get all labels. pub fn labels(&self) -> impl Iterator { self.data.keys() @@ -484,7 +386,6 @@ impl MeasurementsCollection { self.data .values() .flat_map(|samples| samples.keys().copied()) - .filter(|scraper_id| !self.synthetic_only_scrapers.contains(scraper_id)) .collect::>() .len() } @@ -498,133 +399,6 @@ impl MeasurementsCollection { .unwrap_or_default() } - /// Sum a Prometheus counter bucket across scrapers while preserving work - /// observed before a process-local counter reset. - fn sum_count_bucket_increments(&self, label: &str, bucket: &str) -> usize { - self.data - .get(label) - .into_iter() - .flat_map(|by_scraper| by_scraper.values()) - .map(|series| { - let mut previous = 0; - let mut total = 0; - for measurement in series { - let current = measurement.count_buckets.get(bucket).copied().unwrap_or(0); - total += if current >= previous { - current - previous - } else { - current - }; - previous = current; - } - total - }) - .sum() - } - - fn sum_latest_scalar_as_usize(&self, label: &str) -> usize { - self.latest_measurements(label) - .into_iter() - .map(|measurement| measurement.scalar.max(0.0) as usize) - .sum() - } - - /// Sum scalar Prometheus counter increments across scrapers and resets. - fn sum_scalar_counter_increments(&self, label: &str) -> usize { - self.data - .get(label) - .into_iter() - .flat_map(|by_scraper| by_scraper.values()) - .map(|series| { - let mut previous = 0.0; - let mut total = 0.0; - for measurement in series { - let current = measurement.scalar.max(0.0); - total += if current >= previous { - current - previous - } else { - current - }; - previous = current; - } - total as usize - }) - .sum() - } - - fn scraper_series(&self, label: &str, scraper_id: ScraperId) -> Option<&[Measurement]> { - self.data.get(label)?.get(&scraper_id).map(Vec::as_slice) - } - - fn gauge_always_equals(&self, label: &str, scraper_id: ScraperId, expected: f64) -> bool { - self.scraper_series(label, scraper_id) - .is_some_and(|series| { - !series.is_empty() - && series - .iter() - .all(|measurement| measurement.scalar == expected) - }) - } - - fn scalar_counter_is_monotonic_and_positive(&self, label: &str, scraper_id: ScraperId) -> bool { - let Some(series) = self.scraper_series(label, scraper_id) else { - return false; - }; - let monotonic = series - .windows(2) - .all(|window| window[1].scalar >= window[0].scalar); - monotonic - && series - .last() - .is_some_and(|measurement| measurement.scalar > 0.0) - } - - fn count_bucket_is_monotonic_and_positive( - &self, - label: &str, - scraper_id: ScraperId, - bucket: &str, - ) -> bool { - let Some(series) = self.scraper_series(label, scraper_id) else { - return false; - }; - let values = series - .iter() - .map(|measurement| measurement.count_buckets.get(bucket).copied().unwrap_or(0)) - .collect::>(); - values.windows(2).all(|window| window[1] >= window[0]) - && values.last().is_some_and(|value| *value > 0) - } - - fn count_bucket_is_always_zero( - &self, - label: &str, - scraper_id: ScraperId, - bucket: &str, - ) -> bool { - self.scraper_series(label, scraper_id).is_none_or(|series| { - series - .iter() - .all(|measurement| measurement.count_buckets.get(bucket).copied().unwrap_or(0) == 0) - }) - } - - fn latest_scalar_equals(&self, label: &str, scraper_id: ScraperId, expected: f64) -> bool { - self.scraper_series(label, scraper_id) - .and_then(|series| series.last()) - .is_some_and(|measurement| measurement.scalar == expected) - } - - fn gauge_always_at_most(&self, label: &str, scraper_id: ScraperId, maximum: f64) -> bool { - self.scraper_series(label, scraper_id) - .is_some_and(|series| { - !series.is_empty() - && series.iter().all(|measurement| { - measurement.scalar >= 0.0 && measurement.scalar <= maximum - }) - }) - } - /// Aggregate the benchmark duration of multiple data points by taking the /// max. pub fn benchmark_duration(&self) -> Duration { @@ -860,121 +634,6 @@ impl MeasurementsCollection { } }) .collect(); - let shadow_comparison_enabled = self.parameters.consensus_protocol == "starfish-rbc" - && self.parameters.node_parameters.starfish_rbc_dag_shadow; - let shadow_valid_scrapers = self - .data - .get("starfish_rbc_dag_shadow_comparison_valid") - .map(|by_scraper| { - by_scraper - .keys() - .copied() - .filter(|scraper_id| { - self.gauge_always_equals( - "starfish_rbc_dag_shadow_comparison_valid", - *scraper_id, - 1.0, - ) - }) - .collect::>() - }) - .unwrap_or_default(); - let shadow_comparison_valid_nodes = shadow_valid_scrapers.len(); - let shadow_delivery_matches = self.sum_count_bucket_increments( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - "match", - ); - let shadow_delivery_mismatches = ["mismatch", "direct_only", "shadow_only"] - .into_iter() - .map(|bucket| { - self.sum_count_bucket_increments( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - bucket, - ) - }) - .sum(); - let shadow_delivery_ambiguous = self.sum_count_bucket_increments( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - "ambiguous", - ); - let shadow_direct_deliveries = self - .sum_count_bucket_increments("starfish_rbc_dag_shadow_inputs_total", "delivery,direct"); - let shadow_deliveries = self - .sum_count_bucket_increments("starfish_rbc_dag_shadow_inputs_total", "delivery,shadow"); - let shadow_wal_durable_records = - self.sum_scalar_counter_increments("starfish_rbc_dag_shadow_wal_durable_records_total"); - let shadow_pending_recovery = - self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_pending_recovery"); - let shadow_unpaired_direct = - self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_unpaired_direct"); - let shadow_unpaired_shadow = - self.sum_latest_scalar_as_usize("starfish_rbc_dag_shadow_unpaired_shadow"); - let shadow_unpaired_max_round_lag = self.max_result( - "starfish_rbc_dag_shadow_unpaired_max_round_lag", - |measurement| measurement.scalar.max(0.0) as usize, - ); - let maximum_unpaired_per_node = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR - .saturating_mul(i64::try_from(self.parameters.nodes).unwrap_or(i64::MAX)); - let every_shadow_scraper_has_coverage = shadow_valid_scrapers.iter().all(|scraper_id| { - self.count_bucket_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - "delivery,direct", - ) && self.count_bucket_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - "delivery,shadow", - ) && self.count_bucket_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - *scraper_id, - "match", - ) && self.count_bucket_is_always_zero( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - *scraper_id, - "mismatch", - ) && self.count_bucket_is_always_zero( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - *scraper_id, - "direct_only", - ) && self.count_bucket_is_always_zero( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - *scraper_id, - "shadow_only", - ) && self.count_bucket_is_always_zero( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - *scraper_id, - "ambiguous", - ) && self.count_bucket_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_inputs_total", - *scraper_id, - "delivery,shadow", - ) && self.scalar_counter_is_monotonic_and_positive( - "starfish_rbc_dag_shadow_wal_durable_records_total", - *scraper_id, - ) && self.latest_scalar_equals( - "starfish_rbc_dag_shadow_pending_recovery", - *scraper_id, - 0.0, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_unpaired_direct", - *scraper_id, - maximum_unpaired_per_node as f64, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_unpaired_shadow", - *scraper_id, - maximum_unpaired_per_node as f64, - ) && self.gauge_always_at_most( - "starfish_rbc_dag_shadow_unpaired_max_round_lag", - *scraper_id, - STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG as f64, - ) - }); - let expected_shadow_nodes = self.ready_nodes_at_boot(); - let shadow_comparison_valid = shadow_comparison_enabled - && shadow_comparison_valid_nodes == expected_shadow_nodes - && every_shadow_scraper_has_coverage - && shadow_delivery_mismatches == 0 - && shadow_delivery_ambiguous == 0; BenchmarkRunSummary { protocol: self.parameters.consensus_protocol.clone(), @@ -1008,19 +667,6 @@ impl MeasurementsCollection { .average_latest_weighted_scalar("proposed_header_size_bytes"), ready_nodes_at_boot: self.ready_nodes_at_boot(), metrics_contributors: self.metrics_contributors(), - shadow_comparison_enabled, - shadow_comparison_valid, - shadow_comparison_valid_nodes, - shadow_direct_deliveries, - shadow_deliveries, - shadow_delivery_matches, - shadow_delivery_mismatches, - shadow_delivery_ambiguous, - shadow_wal_durable_records, - shadow_pending_recovery, - shadow_unpaired_direct, - shadow_unpaired_shadow, - shadow_unpaired_max_round_lag, } } @@ -1070,29 +716,6 @@ impl MeasurementsCollection { table.add_row(row![b->"Duration:", format!("{:.1} s", duration.as_secs_f64())]); table.add_row(row![b->"TPS:", format!("{:.2} tx/s", summary.tps)]); table.add_row(row![b->"BPS:", format!("{:.2} blocks/s", summary.bps)]); - if summary.shadow_comparison_enabled { - table.add_row(row![ - b->"RBC-DAG shadow:", - format!( - "valid={} ({}/{} validators), direct={}, shadow={}, matches={}, \ - mismatches={}, ambiguous={}, WAL records={}, pending recovery={}, \ - unpaired direct/shadow={}/{}, max unpaired lag={} rounds", - summary.shadow_comparison_valid, - summary.shadow_comparison_valid_nodes, - summary.ready_nodes_at_boot, - summary.shadow_direct_deliveries, - summary.shadow_deliveries, - summary.shadow_delivery_matches, - summary.shadow_delivery_mismatches, - summary.shadow_delivery_ambiguous, - summary.shadow_wal_durable_records, - summary.shadow_pending_recovery, - summary.shadow_unpaired_direct, - summary.shadow_unpaired_shadow, - summary.shadow_unpaired_max_round_lag, - ) - ]); - } table.add_row(row![ b->"End-to-end latency:", format!( @@ -1169,106 +792,6 @@ mod test { use super::{BenchmarkParameters, Measurement, MeasurementsCollection}; use crate::protocol::test_protocol_metrics::TestProtocolMetrics; - fn shadow_benchmark_parameters(nodes: usize) -> BenchmarkParameters { - let mut parameters = BenchmarkParameters::new_for_tests(); - parameters.nodes = nodes; - parameters.consensus_protocol = "starfish-rbc".to_owned(); - parameters.node_parameters.starfish_rbc_dag_shadow = true; - parameters - } - - #[allow(clippy::too_many_arguments)] - fn add_shadow_snapshot( - collection: &mut MeasurementsCollection, - scraper_id: usize, - comparison_valid: f64, - direct_deliveries: usize, - shadow_deliveries: usize, - matches: usize, - mismatches: usize, - direct_only: usize, - shadow_only: usize, - ambiguous: usize, - wal_durable_records: f64, - ) { - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_comparison_valid".to_owned(), - Measurement { - scalar: comparison_valid, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_inputs_total".to_owned(), - Measurement { - count_buckets: HashMap::from([ - ("delivery,direct".to_owned(), direct_deliveries), - ("delivery,shadow".to_owned(), shadow_deliveries), - ]), - count: direct_deliveries + shadow_deliveries, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_delivery_comparisons_total".to_owned(), - Measurement { - count_buckets: HashMap::from([ - ("match".to_owned(), matches), - ("mismatch".to_owned(), mismatches), - ("direct_only".to_owned(), direct_only), - ("shadow_only".to_owned(), shadow_only), - ("ambiguous".to_owned(), ambiguous), - ]), - count: matches + mismatches + direct_only + shadow_only + ambiguous, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_wal_durable_records_total".to_owned(), - Measurement { - count: wal_durable_records as usize, - scalar: wal_durable_records, - ..Measurement::default() - }, - ); - collection.add( - scraper_id, - "starfish_rbc_dag_shadow_pending_recovery".to_owned(), - Measurement::default(), - ); - add_shadow_backlog_snapshot(collection, scraper_id, 0, 0, 0); - } - - fn add_shadow_backlog_snapshot( - collection: &mut MeasurementsCollection, - scraper_id: usize, - unpaired_direct: usize, - unpaired_shadow: usize, - max_round_lag: usize, - ) { - for (label, value) in [ - ("starfish_rbc_dag_shadow_unpaired_direct", unpaired_direct), - ("starfish_rbc_dag_shadow_unpaired_shadow", unpaired_shadow), - ( - "starfish_rbc_dag_shadow_unpaired_max_round_lag", - max_round_lag, - ), - ] { - collection.add( - scraper_id, - label.to_owned(), - Measurement { - scalar: value as f64, - ..Measurement::default() - }, - ); - } - } - #[test] fn average_latency() { let data = Measurement { @@ -1418,199 +941,6 @@ bytes_sent_total 6284648 assert_eq!(data.timestamp.as_secs(), 300); } - #[test] - fn prometheus_parse_preserves_shadow_verdict_and_coverage() { - let report = r#" -# TYPE benchmark_duration counter -benchmark_duration 30 -# TYPE starfish_rbc_dag_shadow_comparison_valid gauge -starfish_rbc_dag_shadow_comparison_valid{node="node-0"} 1 -# TYPE starfish_rbc_dag_shadow_delivery_comparisons_total counter -starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="match"} 7 -starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="mismatch"} 0 -starfish_rbc_dag_shadow_delivery_comparisons_total{node="node-0",outcome="ambiguous"} 0 -# TYPE starfish_rbc_dag_shadow_inputs_total counter -starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="shadow"} 7 -starfish_rbc_dag_shadow_inputs_total{kind="delivery",node="node-0",outcome="direct"} 7 -# TYPE starfish_rbc_dag_shadow_wal_durable_records_total counter -starfish_rbc_dag_shadow_wal_durable_records_total{node="node-0"} 42 -# TYPE starfish_rbc_dag_shadow_wal_replayed_batches gauge -starfish_rbc_dag_shadow_wal_replayed_batches{node="node-0"} 3 -# TYPE starfish_rbc_dag_shadow_pending_recovery gauge -starfish_rbc_dag_shadow_pending_recovery{node="node-0"} 0 -# TYPE starfish_rbc_dag_shadow_unpaired_direct gauge -starfish_rbc_dag_shadow_unpaired_direct{node="node-0"} 2 -# TYPE starfish_rbc_dag_shadow_unpaired_shadow gauge -starfish_rbc_dag_shadow_unpaired_shadow{node="node-0"} 1 -# TYPE starfish_rbc_dag_shadow_unpaired_max_round_lag gauge -starfish_rbc_dag_shadow_unpaired_max_round_lag{node="node-0"} 1 -"#; - - let measurements = Measurement::from_prometheus::(report); - assert_eq!( - measurements["starfish_rbc_dag_shadow_comparison_valid"].scalar, - 1.0 - ); - assert_eq!( - measurements["starfish_rbc_dag_shadow_delivery_comparisons_total"].count_buckets["match"], - 7 - ); - assert_eq!( - measurements["starfish_rbc_dag_shadow_inputs_total"].count_buckets["delivery,shadow"], - 7 - ); - assert_eq!( - measurements["starfish_rbc_dag_shadow_wal_durable_records_total"].scalar, - 42.0 - ); - assert_eq!( - measurements["starfish_rbc_dag_shadow_wal_replayed_batches"].scalar, - 3.0 - ); - - let mut parameters = BenchmarkParameters::new_for_tests(); - parameters.nodes = 1; - parameters.consensus_protocol = "starfish-rbc".to_owned(); - parameters.node_parameters.starfish_rbc_dag_shadow = true; - let mut collection = MeasurementsCollection::new(parameters); - for (label, measurement) in measurements { - collection.add(0, label, measurement); - } - let summary = collection.benchmark_run_summary(); - assert!(summary.shadow_comparison_enabled); - assert!(summary.shadow_comparison_valid); - assert_eq!(summary.shadow_comparison_valid_nodes, 1); - assert_eq!(summary.shadow_direct_deliveries, 7); - assert_eq!(summary.shadow_deliveries, 7); - assert_eq!(summary.shadow_delivery_matches, 7); - assert_eq!(summary.shadow_wal_durable_records, 42); - assert_eq!(summary.shadow_unpaired_direct, 2); - assert_eq!(summary.shadow_unpaired_shadow, 1); - assert_eq!(summary.shadow_unpaired_max_round_lag, 1); - } - - #[test] - fn shadow_verdict_remains_invalid_after_historical_failure_and_counter_reset() { - let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - - // The first scrape records an invalid gauge and every non-match - // comparison category. The second scrape deliberately looks clean, - // including reset comparison counters, so a latest-value-only verdict - // would incorrectly accept the run. - add_shadow_snapshot(&mut collection, 0, 0.0, 1, 1, 1, 1, 1, 1, 1, 1.0); - add_shadow_snapshot(&mut collection, 0, 1.0, 2, 2, 2, 0, 0, 0, 0, 2.0); - - assert!(!collection.gauge_always_equals( - "starfish_rbc_dag_shadow_comparison_valid", - 0, - 1.0, - )); - for bucket in ["mismatch", "direct_only", "shadow_only", "ambiguous"] { - assert!(!collection.count_bucket_is_always_zero( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - 0, - bucket, - )); - } - - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_delivery_mismatches, 3); - assert_eq!(summary.shadow_delivery_ambiguous, 1); - assert!(!summary.shadow_comparison_valid); - } - - #[test] - fn missing_final_shadow_scrape_invalidates_a_previously_valid_snapshot() { - let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); - assert!(collection.benchmark_run_summary().shadow_comparison_valid); - - collection.mark_shadow_final_scrape_missing(0); - - let validity = collection - .scraper_series("starfish_rbc_dag_shadow_comparison_valid", 0) - .unwrap(); - assert_eq!(validity.last().unwrap().scalar_value(), 0.0); - assert!(!collection.benchmark_run_summary().shadow_comparison_valid); - } - - #[test] - fn synthetic_missing_final_marker_is_not_a_metrics_contributor() { - let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - - collection.mark_shadow_final_scrape_missing(0); - - assert_eq!(collection.metrics_contributors(), 0); - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_comparison_valid_nodes, 0); - assert!(!summary.shadow_comparison_valid); - } - - #[test] - fn shadow_verdict_requires_delivery_and_wal_coverage_from_every_scraper() { - let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(2)); - add_shadow_snapshot(&mut collection, 0, 1.0, 4, 4, 4, 0, 0, 0, 0, 8.0); - add_shadow_snapshot(&mut collection, 1, 1.0, 0, 0, 0, 0, 0, 0, 0, 0.0); - - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_comparison_valid_nodes, 2); - assert_eq!(summary.shadow_direct_deliveries, 4); - assert_eq!(summary.shadow_deliveries, 4); - assert_eq!(summary.shadow_delivery_matches, 4); - assert_eq!(summary.shadow_wal_durable_records, 8); - assert!(!summary.shadow_comparison_valid); - } - - #[test] - fn shadow_verdict_accepts_a_bounded_live_pipeline_tail() { - for (direct_deliveries, shadow_deliveries, matches) in [(5, 4, 4), (5, 5, 4)] { - let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot( - &mut collection, - 0, - 1.0, - direct_deliveries, - shadow_deliveries, - matches, - 0, - 0, - 0, - 0, - 8.0, - ); - add_shadow_backlog_snapshot( - &mut collection, - 0, - direct_deliveries - matches, - shadow_deliveries - matches, - 1, - ); - - let summary = collection.benchmark_run_summary(); - assert_eq!(summary.shadow_direct_deliveries, direct_deliveries); - assert_eq!(summary.shadow_deliveries, shadow_deliveries); - assert_eq!(summary.shadow_delivery_matches, matches); - assert!(summary.shadow_comparison_valid); - } - } - - #[test] - fn shadow_verdict_rejects_excessive_or_old_unpaired_work() { - for (unpaired_direct, unpaired_shadow, max_round_lag) in [(5, 0, 1), (0, 5, 1), (1, 0, 5)] { - let mut collection = MeasurementsCollection::new(shadow_benchmark_parameters(1)); - add_shadow_snapshot(&mut collection, 0, 1.0, 8, 7, 7, 0, 0, 0, 0, 8.0); - add_shadow_backlog_snapshot( - &mut collection, - 0, - unpaired_direct, - unpaired_shadow, - max_round_lag, - ); - - assert!(!collection.benchmark_run_summary().shadow_comparison_valid); - } - } - #[test] fn benchmark_run_summary_includes_cpu_and_percentiles() { let report = r#" diff --git a/crates/orchestrator/src/orchestrator.rs b/crates/orchestrator/src/orchestrator.rs index 6ce99bd4..c5a49bf2 100644 --- a/crates/orchestrator/src/orchestrator.rs +++ b/crates/orchestrator/src/orchestrator.rs @@ -398,23 +398,6 @@ impl Orchestrator

{ (node_count as f64 * Self::MAX_TOLERATED_BOOT_FAILURE_RATIO).floor() as usize } - fn max_tolerated_boot_failures_for( - parameters: &BenchmarkParameters, - node_count: usize, - ) -> usize { - if parameters.consensus_protocol == "starfish-rbc" - && parameters.node_parameters.starfish_rbc_dag_shadow - { - // A partial shadow committee cannot produce a complete paired - // comparison. Pre-declared startup faults are already excluded - // through `skipped_node_ids`; every remaining participant must - // finish background replay before measurement begins. - 0 - } else { - Self::max_tolerated_boot_failures(node_count) - } - } - fn apt_get_noninteractive(args: &str) -> String { format!("sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get {args}") } @@ -590,7 +573,7 @@ impl Orchestrator

{ } let timeout = Self::node_boot_timeout(total); - let max_failures = Self::max_tolerated_boot_failures_for(parameters, total); + let max_failures = Self::max_tolerated_boot_failures(total); let start = Instant::now(); display::status(format!("ready 0/{total}")); @@ -1093,13 +1076,6 @@ impl Orchestrator

{ let final_metrics_commands = self .protocol_commands .nodes_final_metrics_command(nodes.clone(), parameters); - let initial_live_scraper_ids: HashSet<_> = nodes - .iter() - .filter(|node| !killed_node_ids.contains(&node.id)) - .filter_map(|node| node_indices.get(&node.id).copied()) - .collect(); - let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" - && parameters.node_parameters.starfish_rbc_dag_shadow; let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1189,7 +1165,6 @@ impl Orchestrator

{ let mut final_instances = final_metrics_commands; final_instances.retain(|(instance, _)| !killed_node_ids.contains(&instance.id)); - let mut fresh_final_shadow_scrapers = HashSet::new(); if !final_instances.is_empty() { let expected_nodes = final_instances.len(); let stdio = self @@ -1200,56 +1175,35 @@ impl Orchestrator

{ ) .await; - if stdio.is_empty() && !shadow_final_scrape_required { + if stdio.is_empty() { display::warn( "Final metrics scrape failed for all reachable nodes; \ reporting the last successful samples", ); } else { - if !shadow_final_scrape_required { - let successful_ids: HashSet<_> = stdio - .iter() - .map(|(instance, _)| instance.id.clone()) - .collect(); - let missed = expected_nodes.saturating_sub(successful_ids.len()); - if missed != 0 { - display::warn(format!( - "Final metrics scrape missed {missed} of {expected_nodes} nodes; \ - reporting partial results", - )); - } + let successful_ids: HashSet<_> = stdio + .iter() + .map(|(instance, _)| instance.id.clone()) + .collect(); + let missed = expected_nodes.saturating_sub(successful_ids.len()); + if missed != 0 { + display::warn(format!( + "Final metrics scrape missed {missed} of {expected_nodes} nodes; \ + reporting partial results", + )); } + for (instance, (stdout, _stderr)) in &stdio { let Some(i) = node_indices.get(&instance.id).copied() else { continue; }; - let parsed = Measurement::from_prometheus::

(stdout); - if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { - fresh_final_shadow_scrapers.insert(i); - } - for (label, measurement) in parsed { + for (label, measurement) in Measurement::from_prometheus::

(stdout) { aggregator.add(i, label, measurement); } } + aggregator.save(&self.suite_results_dir); } } - if shadow_final_scrape_required { - let missed = initial_live_scraper_ids - .difference(&fresh_final_shadow_scrapers) - .count(); - if missed != 0 { - display::warn(format!( - "Final metrics scrape was missing or lacked shadow validity evidence for \ - {missed} of {} initially-live nodes; missing final evidence invalidates stale \ - shadow results", - initial_live_scraper_ids.len(), - )); - } - for scraper_id in initial_live_scraper_ids.difference(&fresh_final_shadow_scrapers) { - aggregator.mark_shadow_final_scrape_missing(*scraper_id); - } - } - aggregator.save(&self.suite_results_dir); display::done(); Ok(aggregator) @@ -1425,13 +1379,6 @@ impl Orchestrator

{ let final_metrics_commands = self .protocol_commands .nodes_final_metrics_command(nodes.clone(), parameters); - let initial_live_scraper_ids: HashSet<_> = nodes - .iter() - .filter(|node| !killed_node_ids.contains(&node.id)) - .filter_map(|node| node_indices.get(&node.id).copied()) - .collect(); - let shadow_final_scrape_required = parameters.consensus_protocol == "starfish-rbc" - && parameters.node_parameters.starfish_rbc_dag_shadow; let mut aggregator = MeasurementsCollection::new(parameters.clone()); aggregator.set_ready_nodes_at_boot(nodes.len().saturating_sub(killed_node_ids.len())); @@ -1812,9 +1759,7 @@ impl Orchestrator

{ let mut final_instances = final_metrics_commands; final_instances.retain(|(instance, _)| !killed_node_ids.contains(&instance.id)); - let mut fresh_final_shadow_scrapers = HashSet::new(); if !final_instances.is_empty() { - let expected_nodes = final_instances.len(); let stdio = self .execute_per_instance_best_effort( final_instances, @@ -1822,54 +1767,16 @@ impl Orchestrator

{ "Final stability metrics scrape", ) .await; - if stdio.is_empty() && !shadow_final_scrape_required { - display::warn( - "Final stability metrics scrape failed for all reachable nodes; \ - reporting the last successful samples", - ); - } else if !shadow_final_scrape_required { - let successful_ids: HashSet<_> = stdio - .iter() - .map(|(instance, _)| instance.id.clone()) - .collect(); - let missed = expected_nodes.saturating_sub(successful_ids.len()); - if missed != 0 { - display::warn(format!( - "Final stability metrics scrape missed {missed} of {expected_nodes} nodes; \ - reporting partial results", - )); - } - } for (instance, (stdout, _stderr)) in &stdio { let Some(i) = node_indices.get(&instance.id).copied() else { continue; }; - let parsed = Measurement::from_prometheus::

(stdout); - if parsed.contains_key("starfish_rbc_dag_shadow_comparison_valid") { - fresh_final_shadow_scrapers.insert(i); - } - for (label, measurement) in parsed { + for (label, measurement) in Measurement::from_prometheus::

(stdout) { aggregator.add(i, label, measurement); } } + aggregator.save(&self.suite_results_dir); } - if shadow_final_scrape_required { - let missed = initial_live_scraper_ids - .difference(&fresh_final_shadow_scrapers) - .count(); - if missed != 0 { - display::warn(format!( - "Final stability scrape was missing or lacked shadow validity evidence for \ - {missed} of {} initially-live nodes; missing final evidence invalidates stale \ - shadow results", - initial_live_scraper_ids.len(), - )); - } - for scraper_id in initial_live_scraper_ids.difference(&fresh_final_shadow_scrapers) { - aggregator.mark_shadow_final_scrape_missing(*scraper_id); - } - } - aggregator.save(&self.suite_results_dir); display::done(); Ok((aggregator, report)) @@ -2294,7 +2201,6 @@ impl Orchestrator

{ #[cfg(test)] mod tests { use super::Orchestrator; - use crate::benchmark::BenchmarkParameters; use crate::protocol::starfish::StarfishProtocol; use crate::{client::Instance, faults::FaultsType}; @@ -2326,18 +2232,6 @@ mod tests { ); } - #[test] - fn shadow_readiness_requires_every_non_skipped_validator() { - let mut parameters = BenchmarkParameters::new_for_tests(); - parameters.consensus_protocol = "starfish-rbc".to_owned(); - parameters.node_parameters.starfish_rbc_dag_shadow = true; - - assert_eq!( - Orchestrator::::max_tolerated_boot_failures_for(¶meters, 10), - 0 - ); - } - #[test] fn startup_permanent_faults_are_left_down_from_boot() { let nodes = (0..100) diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index e5bcdf1f..11a7fe72 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -15,10 +15,7 @@ use starfish_core::{ types::AuthorityIndex, }; -use super::{ - BINARY_PATH, METRICS_CURL_CONNECT_TIMEOUT_SECS, METRICS_CURL_MAX_TIME_SECS, ProtocolCommands, - ProtocolMetrics, ProtocolParameters, -}; +use super::{BINARY_PATH, ProtocolCommands, ProtocolMetrics, ProtocolParameters}; use crate::{benchmark::BenchmarkParameters, client::Instance, settings::Settings}; #[derive(Clone, Serialize, Deserialize, Default)] @@ -247,50 +244,6 @@ impl ProtocolMetrics for StarfishProtocol { instances.into_iter().zip(metrics_paths).collect() } - - fn nodes_readiness_command( - &self, - instances: I, - parameters: &BenchmarkParameters, - ) -> Vec<(Instance, String)> - where - I: IntoIterator, - { - if parameters.consensus_protocol != "starfish-rbc" - || !parameters.node_parameters.starfish_rbc_dag_shadow - { - return self - .nodes_metrics_path(instances, parameters) - .into_iter() - .map(|(instance, path)| { - ( - instance, - format!( - "curl -sf -o /dev/null --compressed --connect-timeout \ - {METRICS_CURL_CONNECT_TIMEOUT_SECS} --max-time \ - {METRICS_CURL_MAX_TIME_SECS} {path}" - ), - ) - }) - .collect(); - } - - self.nodes_metrics_path(instances, parameters) - .into_iter() - .map(|(instance, path)| { - ( - instance, - format!( - "curl --silent --show-error --fail --compressed --connect-timeout \ - {METRICS_CURL_CONNECT_TIMEOUT_SECS} --max-time \ - {METRICS_CURL_MAX_TIME_SECS} {path} | grep -Eq \ - '^starfish_rbc_dag_shadow_comparison_valid(\\{{[^}}]*\\}})? \ - 1(\\.0)?$'" - ), - ) - }) - .collect() - } } impl StarfishProtocol { @@ -312,12 +265,6 @@ impl StarfishProtocol { ) -> StarfishNodeParameters { if consensus_protocol == "starfish-rbc" { node_parameters.refresh_starfish_rbc_protocol_instance(); - } else { - // The CLI flag is global to a multi-protocol benchmark plan, but - // the shadow is meaningful only for the Starfish-RBC member. Do - // not let it make Sailfish++ or another comparison member fail - // validator configuration. - node_parameters.starfish_rbc_dag_shadow = false; } node_parameters } @@ -352,57 +299,28 @@ impl StarfishProtocol { #[cfg(test)] mod tests { - use super::{ProtocolMetrics, StarfishNodeParameters, StarfishProtocol}; - use crate::{benchmark::BenchmarkParameters, client::Instance}; - use starfish_core::config::NodeParameters; + use super::{StarfishNodeParameters, StarfishProtocol}; #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { - let shared_parameters = StarfishNodeParameters(NodeParameters { - starfish_rbc_dag_shadow: true, - ..NodeParameters::default() - }); - let parameters = - StarfishProtocol::node_parameters_for_genesis("starfish-rbc", shared_parameters); + let parameters = StarfishProtocol::node_parameters_for_genesis( + "starfish-rbc", + StarfishNodeParameters::default(), + ); assert!( parameters .starfish_rbc_protocol_instance .is_some_and(|instance| instance != [0; 32]) ); - assert!(parameters.starfish_rbc_dag_shadow); } #[test] fn non_rbc_genesis_does_not_need_a_protocol_instance() { - let shared_parameters = StarfishNodeParameters(NodeParameters { - starfish_rbc_dag_shadow: true, - ..NodeParameters::default() - }); - let parameters = - StarfishProtocol::node_parameters_for_genesis("sailfish++", shared_parameters); - assert_eq!(parameters.starfish_rbc_protocol_instance, None); - assert!( - !parameters.starfish_rbc_dag_shadow, - "a global shadow flag must not leak into non-RBC comparison members" + let parameters = StarfishProtocol::node_parameters_for_genesis( + "starfish", + StarfishNodeParameters::default(), ); - } - - #[test] - fn shadow_readiness_waits_for_completed_background_replay() { - let protocol = StarfishProtocol { - working_dir: std::path::PathBuf::from("benchmark"), - }; - let mut parameters = BenchmarkParameters::new_for_tests(); - parameters.consensus_protocol = "starfish-rbc".to_owned(); - parameters.node_parameters.starfish_rbc_dag_shadow = true; - let command = protocol - .nodes_readiness_command(vec![Instance::new_for_test("1".into())], ¶meters) - .pop() - .unwrap() - .1; - - assert!(command.contains("starfish_rbc_dag_shadow_comparison_valid")); - assert!(command.contains("grep -Eq")); + assert_eq!(parameters.starfish_rbc_protocol_instance, None); } #[test] diff --git a/crates/starfish-core/src/config.rs b/crates/starfish-core/src/config.rs index 5910be63..58d403a6 100644 --- a/crates/starfish-core/src/config.rs +++ b/crates/starfish-core/src/config.rs @@ -67,11 +67,6 @@ pub struct NodeParameters { /// other protocols. #[serde(default)] pub starfish_rbc_protocol_instance: Option<[u8; 32]>, - /// Run the persisted Starfish-RBC-DAG carrier implementation alongside - /// the authoritative direct Starfish-RBC service. Shadow delivery is - /// observational only and cannot affect the DAG, pacemaker, or commits. - #[serde(default)] - pub starfish_rbc_dag_shadow: bool, /// Testbed-only receiver-local single-DAG RBC path: deliver an exact header /// after locally observing quorum ECHO rather than quorum READY. Quorum /// intersection preserves a unique value, but pairwise-MAC testimony is @@ -154,7 +149,6 @@ impl Default for NodeParameters { dissemination_mode: DisseminationMode::default(), block_authentication: None, starfish_rbc_protocol_instance: None, - starfish_rbc_dag_shadow: false, starfish_rbc_single_dag_echo_qc_fast_path: false, causal_push_shard_round_lag: node_defaults::default_causal_push_shard_round_lag(), enable_strong_vote_adaptive_acknowledgments: @@ -389,10 +383,6 @@ impl NodePrivateConfig { pub fn rocksdb(&self) -> PathBuf { self.storage_path.join("rocksdb") } - - pub fn starfish_rbc_dag_shadow_wal(&self) -> PathBuf { - self.storage_path.join("starfish-rbc-dag-shadow-v1.wal") - } } impl ImportExport for NodePrivateConfig {} @@ -405,7 +395,6 @@ mod tests { fn starfish_rbc_protocol_instance_is_optional_and_roundtrips() { let mut parameters: NodeParameters = serde_yaml::from_str("{}").unwrap(); assert_eq!(parameters.starfish_rbc_protocol_instance, None); - assert!(!parameters.starfish_rbc_dag_shadow); assert!(!parameters.starfish_rbc_single_dag_echo_qc_fast_path); let protocol_instance = parameters.refresh_starfish_rbc_protocol_instance(); @@ -417,7 +406,6 @@ mod tests { decoded.starfish_rbc_protocol_instance, Some(protocol_instance) ); - assert!(!decoded.starfish_rbc_dag_shadow); assert!(!decoded.starfish_rbc_single_dag_echo_qc_fast_path); } } diff --git a/crates/starfish-core/src/core_thread/spawned.rs b/crates/starfish-core/src/core_thread/spawned.rs index 141b9a08..450315f9 100644 --- a/crates/starfish-core/src/core_thread/spawned.rs +++ b/crates/starfish-core/src/core_thread/spawned.rs @@ -524,16 +524,7 @@ mod tests { recovered, None, ); - let syncer = Syncer::new( - core, - false, - NoopCommitObserver, - metrics, - None, - None, - None, - None, - ); + let syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None, None); CoreThreadDispatcher::start(syncer) } diff --git a/crates/starfish-core/src/lib.rs b/crates/starfish-core/src/lib.rs index 15dee4b7..70c7f628 100644 --- a/crates/starfish-core/src/lib.rs +++ b/crates/starfish-core/src/lib.rs @@ -31,8 +31,6 @@ mod runtime; pub mod shard_reconstructor; pub mod starfish_rbc; pub mod starfish_rbc_dag; -mod starfish_rbc_dag_shadow; -mod starfish_rbc_dag_shadow_service; mod starfish_rbc_service; mod stat; mod state; diff --git a/crates/starfish-core/src/metrics.rs b/crates/starfish-core/src/metrics.rs index 83487543..15004578 100644 --- a/crates/starfish-core/src/metrics.rs +++ b/crates/starfish-core/src/metrics.rs @@ -36,11 +36,6 @@ pub const BENCHMARK_DURATION: &str = "benchmark_duration"; pub const TRANSACTION_CERTIFIED_LATENCY: &str = "transaction_certified_latency"; pub const TRANSACTION_CERTIFIED_LATENCY_SQUARED: &str = "latency_s"; -/// Benchmark-only live-tail guards for the observational RBC-DAG shadow. -/// They are not asynchronous protocol or garbage-collection bounds. -pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR: i64 = 4; -pub const STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG: i64 = 4; - #[derive(Clone)] pub struct Metrics { pub benchmark_duration: IntCounter, @@ -140,21 +135,6 @@ pub struct Metrics { pub network_message_bytes_sent_total: IntCounterVec, pub network_message_bytes_received_total: IntCounterVec, - // Starfish-RBC-DAG shadow instrumentation. These metrics are strictly - // observational: the shadow path never feeds the authoritative DAG or - // consensus state. - pub starfish_rbc_dag_shadow_inputs_total: IntCounterVec, - pub starfish_rbc_dag_shadow_delivery_comparisons_total: IntCounterVec, - pub starfish_rbc_dag_shadow_wal_durable_batches_total: IntCounter, - pub starfish_rbc_dag_shadow_wal_durable_records_total: IntCounter, - pub starfish_rbc_dag_shadow_wal_replayed_batches: IntGauge, - pub starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total: IntCounter, - pub starfish_rbc_dag_shadow_pending_recovery: IntGauge, - pub starfish_rbc_dag_shadow_unpaired_direct: IntGauge, - pub starfish_rbc_dag_shadow_unpaired_shadow: IntGauge, - pub starfish_rbc_dag_shadow_unpaired_max_round_lag: IntGauge, - pub starfish_rbc_dag_shadow_comparison_valid: IntGauge, - // subscription tracking pub subscribed_to_peers: IntGauge, pub subscribed_by_peers: IntGauge, @@ -536,76 +516,6 @@ impl Metrics { registry, ) .unwrap(), - starfish_rbc_dag_shadow_inputs_total: register_int_counter_vec_with_registry!( - "starfish_rbc_dag_shadow_inputs_total", - "Starfish-RBC-DAG shadow inputs, by bounded input kind and processing outcome", - &["kind", "outcome"], - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_delivery_comparisons_total: - register_int_counter_vec_with_registry!( - "starfish_rbc_dag_shadow_delivery_comparisons_total", - "Non-authoritative current-process paired direct-vs-shadow delivery observations, by outcome; unmatched observations are not mismatches", - &["outcome"], - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_wal_durable_batches_total: register_int_counter_with_registry!( - "starfish_rbc_dag_shadow_wal_durable_batches_total", - "Starfish-RBC-DAG shadow WAL batches durably synchronized", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_wal_durable_records_total: register_int_counter_with_registry!( - "starfish_rbc_dag_shadow_wal_durable_records_total", - "Starfish-RBC-DAG shadow WAL records durably synchronized", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_wal_replayed_batches: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_wal_replayed_batches", - "Starfish-RBC-DAG shadow WAL batches replayed during this process startup", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total: - register_int_counter_with_registry!( - "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total", - "Physically incomplete final Starfish-RBC-DAG shadow WAL bytes discarded during recovery", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_pending_recovery: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_pending_recovery", - "Current Starfish-RBC-DAG shadow carrier-content recoveries pending", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_unpaired_direct: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_unpaired_direct", - "Current direct-delivery slots without an observed shadow-delivery slot, including recovered shadow state", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_unpaired_shadow: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_unpaired_shadow", - "Current shadow-delivery slots first observed in this process without a direct-delivery slot observed in this process", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_unpaired_max_round_lag: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_unpaired_max_round_lag", - "Maximum round lag from a current unpaired direct or current-epoch shadow delivery slot to the newest current-process observation", - registry, - ) - .unwrap(), - starfish_rbc_dag_shadow_comparison_valid: register_int_gauge_with_registry!( - "starfish_rbc_dag_shadow_comparison_valid", - "State of the non-authoritative shadow observation stream (1 valid, 0 disabled/invalid, -1 starting)", - registry, - ) - .unwrap(), subscribed_to_peers: register_int_gauge_with_registry!( "subscribed_to_peers", "Number of peers this validator is subscribed to", @@ -1062,8 +972,6 @@ impl Metrics { metrics: Vec>, reporters: Vec>, duration_secs: u64, - committee_size: usize, - starfish_rbc_dag_shadow_expected: bool, ) { let num_validators = metrics.len() as u64; @@ -1201,9 +1109,6 @@ impl Metrics { "rbc_ready", "rbc_header_request", "rbc_header_response", - "rbc_dag_shadow_carrier", - "rbc_dag_shadow_carrier_request", - "rbc_dag_shadow_carrier_response", ]; let outbound_message_breakdown = NETWORK_MESSAGE_TYPES .iter() @@ -1259,154 +1164,6 @@ impl Metrics { }; table.add_row(row![b->"Bandwidth efficiency:", format!("{:.2}", bandwidth_efficiency)]); - if starfish_rbc_dag_shadow_expected { - let valid_nodes = metrics - .iter() - .filter(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) - .count(); - let matches = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["match"]) - .get() - }) - .sum::(); - let mismatches = metrics - .iter() - .map(|metrics| { - ["mismatch", "direct_only", "shadow_only"] - .into_iter() - .map(|outcome| { - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&[outcome]) - .get() - }) - .sum::() - }) - .sum::(); - let ambiguous = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["ambiguous"]) - .get() - }) - .sum::(); - let shadow_deliveries = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get() - }) - .sum::(); - let direct_deliveries = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "direct"]) - .get() - }) - .sum::(); - let wal_records = metrics - .iter() - .map(|metrics| { - metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .get() - }) - .sum::(); - let pending_recovery = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_pending_recovery.get()) - .sum::(); - let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR - .saturating_mul(i64::try_from(committee_size).unwrap_or(i64::MAX)); - let every_node_has_exact_coverage = metrics.iter().all(|metrics| { - let direct = metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "direct"]) - .get(); - let shadow = metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get(); - let matched = metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["match"]) - .get(); - direct > 0 - && shadow > 0 - && matched > 0 - && metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .get() - > 0 - && metrics.starfish_rbc_dag_shadow_unpaired_direct.get() <= maximum_unpaired - && metrics.starfish_rbc_dag_shadow_unpaired_shadow.get() <= maximum_unpaired - && metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get() - <= STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG - }); - let comparison_valid = valid_nodes == metrics.len() - && every_node_has_exact_coverage - && mismatches == 0 - && ambiguous == 0 - && wal_records > 0 - && pending_recovery == 0; - - table.add_row(row![bH2->""]); - table.add_row(row![bH2->"RBC-DAG Shadow Verification"]); - table.add_row(row![ - b->"Comparison verdict:", - if comparison_valid { - "VALID".to_owned() - } else { - "INVALID — DISCARD THIS SHADOW COMPARISON".to_owned() - } - ]); - table.add_row(row![ - b->"Valid validators:", - format!("{valid_nodes}/{}", metrics.len()) - ]); - table.add_row(row![ - b->"Paired deliveries:", - format!( - "direct={direct_deliveries}, shadow={shadow_deliveries}, matches={matches}, \ - mismatches={mismatches}, ambiguous={ambiguous}" - ) - ]); - table.add_row(row![ - b->"Durability/recovery:", - format!("WAL records={wal_records}, pending recovery={pending_recovery}") - ]); - let unpaired_direct = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_unpaired_direct.get()) - .sum::(); - let unpaired_shadow = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_unpaired_shadow.get()) - .sum::(); - let max_unpaired_lag = metrics - .iter() - .map(|metrics| metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get()) - .max() - .unwrap_or_default(); - table.add_row(row![ - b->"Live comparison tail:", - format!( - "unpaired direct/shadow={unpaired_direct}/{unpaired_shadow}, \ - max lag={max_unpaired_lag} rounds" - ) - ]); - } - // Shard reconstruction metrics table.add_row(row![bH2->""]); table.add_row(row![bH2->"Shard Reconstruction"]); @@ -1705,60 +1462,3 @@ struct NetworkAddressTable { peer: String, address: String, } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn registers_starfish_rbc_dag_shadow_metrics() { - let registry = Registry::new(); - let (metrics, _reporter) = Metrics::new(®istry, None, None, None); - - metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["authenticated_ingress", "accepted"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["match"]) - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .inc(); - metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .inc_by(3); - metrics.starfish_rbc_dag_shadow_wal_replayed_batches.set(4); - metrics - .starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total - .inc_by(5); - metrics.starfish_rbc_dag_shadow_pending_recovery.set(2); - metrics.starfish_rbc_dag_shadow_unpaired_direct.set(6); - metrics.starfish_rbc_dag_shadow_unpaired_shadow.set(7); - metrics - .starfish_rbc_dag_shadow_unpaired_max_round_lag - .set(8); - metrics.starfish_rbc_dag_shadow_comparison_valid.set(1); - - let gathered = registry.gather(); - for name in [ - "starfish_rbc_dag_shadow_inputs_total", - "starfish_rbc_dag_shadow_delivery_comparisons_total", - "starfish_rbc_dag_shadow_wal_durable_batches_total", - "starfish_rbc_dag_shadow_wal_durable_records_total", - "starfish_rbc_dag_shadow_wal_replayed_batches", - "starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total", - "starfish_rbc_dag_shadow_pending_recovery", - "starfish_rbc_dag_shadow_unpaired_direct", - "starfish_rbc_dag_shadow_unpaired_shadow", - "starfish_rbc_dag_shadow_unpaired_max_round_lag", - "starfish_rbc_dag_shadow_comparison_valid", - ] { - assert!( - gathered.iter().any(|family| family.get_name() == name), - "metric family {name} was not registered", - ); - } - } -} diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index f5342d8f..226beaee 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -4,7 +4,6 @@ use std::{ collections::{HashMap, VecDeque}, - path::PathBuf, sync::{ Arc, atomic::{AtomicU32, Ordering}, @@ -46,14 +45,6 @@ use crate::{ }, shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, - starfish_rbc_dag::{RbcDagCommitteeContextV1, RbcDagContextV1, RbcDagProtocolInstanceId}, - starfish_rbc_dag_shadow::{ - ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, - }, - starfish_rbc_dag_shadow_service::{ - ShadowServiceErrorV1, ShadowServiceEventV1, StarfishRbcDagShadowServiceHandleV1, - start_starfish_rbc_dag_shadow_service_v1, - }, starfish_rbc_service::{ RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, }, @@ -69,82 +60,6 @@ const MAX_FILTER_SIZE: usize = 100_000; const SAILFISH_CERT_BATCH_FLUSH_INTERVAL: Duration = Duration::from_millis(5); const SAILFISH_CERT_BATCH_MAX_LEN: usize = 256; const STARFISH_RBC_HEADER_RETRY_INTERVAL: Duration = Duration::from_millis(250); -const STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); - -/// Recover the exact locally selected Starfish-RBC chain so the persisted -/// non-authoritative shadow can reconcile a WAL that ended before the direct -/// DAG. The newest local block determines the branch when a Byzantine test -/// produced more than one value in a round. -fn recovered_local_rbc_headers( - core: &Core, -) -> Result, String> { - if !core.dag_state().consensus_protocol.is_starfish_rbc() || core.last_proposed() == 0 { - return Ok(Vec::new()); - } - - let own_authority = core.authority(); - let store = core.store(); - let mut current = core.last_own_block().clone(); - let mut reversed = Vec::with_capacity(current.round() as usize); - loop { - if current.round() == 0 { - break; - } - if current.authority() != own_authority { - return Err(format!( - "recovered local chain contains authority {} at round {} (expected {})", - current.authority(), - current.round(), - own_authority - )); - } - reversed.push( - RbcCanonicalHeader::from_block_header(current.header()).map_err(|error| { - format!( - "recovered local Starfish-RBC header {} is not canonical: {error}", - current.reference() - ) - })?, - ); - if current.round() == 1 { - break; - } - - let expected_round = current.round() - 1; - let predecessor = current - .block_references() - .iter() - .find(|reference| { - reference.authority == own_authority && reference.round == expected_round - }) - .copied() - .ok_or_else(|| { - format!( - "recovered local Starfish-RBC block {} has no own predecessor at round {}", - current.reference(), - expected_round - ) - })?; - current = core - .dag_state() - .get_blocks_at_authority_round(own_authority, expected_round) - .into_iter() - .find(|block| block.reference() == &predecessor) - .or_else(|| store.get_block(&predecessor).ok().flatten()) - .ok_or_else(|| { - format!("recovered local Starfish-RBC predecessor {predecessor} is unavailable") - })?; - } - reversed.reverse(); - Ok(reversed) -} - -fn shadow_transport_error_invalidates_comparison(error: &ShadowServiceErrorV1) -> bool { - matches!( - error, - ShadowServiceErrorV1::Overloaded { .. } | ShadowServiceErrorV1::Stopped - ) -} /// Enforce the MAC experiment's transport contract before cryptographic /// verification: @@ -826,7 +741,6 @@ struct ConnectionHandler bls_service: Option, sailfish_service: Option, starfish_rbc_service: Option, - starfish_rbc_dag_shadow_service: Option, } impl ConnectionHandler { @@ -869,7 +783,6 @@ impl ConnectionHandler ConnectionHandler ConnectionHandler { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier(self.peer_id, envelope) { - if shadow_transport_error_invalidates_comparison(&error) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - } - tracing::warn!("Failed to forward RBC-DAG shadow carrier: {error}"); - } - } - } - NetworkMessage::RbcDagShadowCarrierRequest(reference) => { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_request(self.peer_id, reference) { - if shadow_transport_error_invalidates_comparison(&error) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - } - tracing::warn!("Failed to forward RBC-DAG shadow request: {error}"); - } - } - } - NetworkMessage::RbcDagShadowCarrierResponse(response) => { - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.carrier_response(self.peer_id, response) { - if shadow_transport_error_invalidates_comparison(&error) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - } - tracing::warn!("Failed to forward RBC-DAG shadow response: {error}"); - } - } - } } true } @@ -1684,8 +1566,6 @@ pub struct NetworkSyncer { sf_event_task: Option>, rbc_event_task: Option>, rbc_service_task: Option>, - rbc_dag_shadow_event_task: Option>, - rbc_dag_shadow_service_task: Option>, cordial_knowledge_task: JoinHandle<()>, } @@ -1724,9 +1604,6 @@ pub struct NetworkSyncerInner { /// Central Starfish-RBC service. Connection workers only forward their /// trusted peer identity and wire payload into this single owner. pub(crate) starfish_rbc_service: Option, - /// Non-authoritative persisted embedded-RBC shadow. It emits only network - /// and metric events and can never call the core dispatcher. - pub(crate) starfish_rbc_dag_shadow_service: Option, /// Wall-clock at NetworkSyncer start; consumed by time-dependent /// Byzantine strategies (e.g. RampUpWithholding) to ramp behavior /// over a fixed schedule. @@ -1740,7 +1617,6 @@ impl NetworkSyncer mut commit_observer: C, metrics: Arc, node_parameters: NodeParameters, - starfish_rbc_dag_shadow_wal: PathBuf, partial_sig_outbox_rx: Option>, bls_cert_aggregator: Option, bls_signer: Option, @@ -1753,22 +1629,6 @@ impl NetworkSyncer let committee = core.committee().clone(); let mac_keys = core.mac_keys(); let dag_state = core.dag_state().clone(); - let recovered_shadow_local_headers = if node_parameters.starfish_rbc_dag_shadow { - match recovered_local_rbc_headers(&core) { - Ok(headers) => Some(headers), - Err(error) => { - // A partial local history would make delivery comparisons - // meaningless. Disable the whole observational run while - // allowing authoritative direct RBC to continue. - tracing::error!( - "Disabling non-authoritative RBC-DAG shadow because recovered direct headers cannot be reconciled: {error}" - ); - None - } - } - } else { - None - }; let dissemination_mode = dag_state .consensus_protocol .resolve_dissemination_mode(node_parameters.dissemination_mode); @@ -1841,57 +1701,6 @@ impl NetworkSyncer } else { (None, None, None) }; - let (starfish_rbc_dag_shadow_service, rbc_dag_shadow_event_rx, rbc_dag_shadow_service_task) = - if let Some(recovered_local_headers) = recovered_shadow_local_headers { - let protocol_instance_bytes = node_parameters - .starfish_rbc_protocol_instance - .expect("validated shadow configuration must share the direct RBC instance"); - let protocol_instance = RbcDagProtocolInstanceId::new(protocol_instance_bytes) - .expect("validated direct RBC instance must be nonzero"); - let committee_context = RbcDagCommitteeContextV1::new(committee.clone()) - .expect("validated committee must initialize the RBC-DAG shadow"); - let context = RbcDagContextV1::new_with_committee( - protocol_instance, - &committee_context, - dag_state.block_authentication_scheme, - ); - let authorizer = match dag_state.block_authentication_scheme { - BlockAuthenticationScheme::Ed25519 => { - ShadowAuthorizerV1::Ed25519(core.get_signer().clone()) - } - BlockAuthenticationScheme::MlDsa44 => { - ShadowAuthorizerV1::MlDsa44(core.get_ml_dsa_44_signer().clone()) - } - BlockAuthenticationScheme::MlDsa65 => { - ShadowAuthorizerV1::MlDsa65(core.get_ml_dsa_65_signer().clone()) - } - BlockAuthenticationScheme::MacVector => { - ShadowAuthorizerV1::MacVector(mac_keys.as_ref().clone()) - } - }; - // -1 means the background WAL replay has not completed yet; - // Ready moves this to 1 unless work was already shed (0). - metrics.starfish_rbc_dag_shadow_comparison_valid.set(-1); - match start_starfish_rbc_dag_shadow_service_v1( - starfish_rbc_dag_shadow_wal, - committee_context, - dag_state.get_own_authority_index(), - context, - authorizer, - recovered_local_headers, - ) { - Ok((service, events, task)) => (Some(service), Some(events), Some(task)), - Err(error) => { - metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - tracing::error!( - "Disabling non-authoritative Starfish-RBC-DAG shadow: {error}" - ); - (None, None, None) - } - } - } else { - (None, None, None) - }; let syncer = Syncer::new( core, NetworkSyncSignals { @@ -1903,7 +1712,6 @@ impl NetworkSyncer bls_msg_tx.clone(), sf_msg_tx.clone(), starfish_rbc_service.clone(), - starfish_rbc_dag_shadow_service.clone(), ); let initial_round = syncer.core().next_block_round(); let syncer = CoreThreadDispatcher::start(syncer); @@ -1973,7 +1781,6 @@ impl NetworkSyncer soft_block_timeout: node_parameters.soft_block_timeout, sailfish_handle: sf_handle_for_inner, starfish_rbc_service: starfish_rbc_service.clone(), - starfish_rbc_dag_shadow_service: starfish_rbc_dag_shadow_service.clone(), start_time: std::time::Instant::now(), }); @@ -1983,7 +1790,6 @@ impl NetworkSyncer // clean. let rbc_event_task = rbc_event_rx.map(|mut event_rx| { let event_inner = inner.clone(); - let rbc_metrics = metrics.clone(); handle.spawn(async move { let mut payload_encoder = ReedSolomonEncoder::new(2, 4, 2) .expect("Starfish-RBC payload encoder should be created"); @@ -2075,26 +1881,6 @@ impl NetworkSyncer .await; } RbcServiceEvent::Delivered(header) => { - if let Some(ref shadow) = - event_inner.starfish_rbc_dag_shadow_service - { - let canonical = header.header(); - let identity = ShadowDeliveryIdentityV1::new( - canonical.reference().authority, - canonical.reference().round, - canonical.transactions_commitment(), - ); - if let Err(error) = shadow.direct_delivered(identity) { - if shadow_transport_error_invalidates_comparison(&error) { - rbc_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - tracing::warn!( - "Failed to notify RBC-DAG shadow of direct delivery: {error}" - ); - } - } event_inner .syncer .apply_starfish_rbc_deliveries(vec![header]) @@ -2118,155 +1904,6 @@ impl NetworkSyncer }) }); - let rbc_dag_shadow_event_task = rbc_dag_shadow_event_rx.map(|mut event_rx| { - let event_inner = inner.clone(); - let shadow_metrics = metrics.clone(); - handle.spawn(async move { - while let Some(event) = event_rx.recv().await { - match event { - ShadowServiceEventV1::Network { recipient, message } => { - let sender = event_inner.peer_senders.read().get(&recipient).cloned(); - if let Some(sender) = sender { - match sender.try_send(message) { - Ok(()) => shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "sent"]) - .inc(), - Err(mpsc::error::TrySendError::Full(_)) => { - // The shadow is observational. It must - // shed work instead of backpressuring - // the authoritative network path. - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "dropped_backpressure"]) - .inc(); - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "disconnected"]) - .inc(); - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - } - } else { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["network", "disconnected"]) - .inc(); - // No observation was lost: the actor retains - // local carriers and replays them when this - // peer connects. - } - } - ShadowServiceEventV1::Delivered(identity) => { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .inc(); - tracing::debug!(?identity, "RBC-DAG shadow delivered carrier"); - } - ShadowServiceEventV1::ComparisonBacklog { - unpaired_direct, - unpaired_shadow, - max_round_lag, - } => { - shadow_metrics - .starfish_rbc_dag_shadow_unpaired_direct - .set(i64::try_from(unpaired_direct).unwrap_or(i64::MAX)); - shadow_metrics - .starfish_rbc_dag_shadow_unpaired_shadow - .set(i64::try_from(unpaired_shadow).unwrap_or(i64::MAX)); - shadow_metrics - .starfish_rbc_dag_shadow_unpaired_max_round_lag - .set(i64::from(max_round_lag)); - } - ShadowServiceEventV1::Comparison(comparison) => { - let outcome = match &comparison { - ShadowDeliveryComparisonV1::Match => "match", - ShadowDeliveryComparisonV1::Mismatch { - direct_only, - shadow_only, - } if !direct_only.is_empty() && shadow_only.is_empty() => { - "direct_only" - } - ShadowDeliveryComparisonV1::Mismatch { - direct_only, - shadow_only, - } if direct_only.is_empty() && !shadow_only.is_empty() => { - "shadow_only" - } - ShadowDeliveryComparisonV1::Mismatch { .. } => "mismatch", - ShadowDeliveryComparisonV1::Ambiguous { .. } => "ambiguous", - }; - shadow_metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&[outcome]) - .inc(); - } - ShadowServiceEventV1::Input { kind, outcome } => { - shadow_metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&[kind, outcome]) - .inc(); - } - ShadowServiceEventV1::WalDurable { batches, records } => { - shadow_metrics - .starfish_rbc_dag_shadow_wal_durable_batches_total - .inc_by(batches); - shadow_metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .inc_by(records); - } - ShadowServiceEventV1::Ready => { - if shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .get() - != 0 - { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(1); - } - } - ShadowServiceEventV1::Recovered { - batches, - discarded_tail_bytes, - } => { - shadow_metrics - .starfish_rbc_dag_shadow_wal_replayed_batches - .set(batches as i64); - shadow_metrics - .starfish_rbc_dag_shadow_wal_discarded_tail_bytes_total - .inc_by(discarded_tail_bytes); - } - ShadowServiceEventV1::PendingRecovery(pending) => { - shadow_metrics - .starfish_rbc_dag_shadow_pending_recovery - .set(pending as i64); - } - ShadowServiceEventV1::Rejected { peer, error } => { - if peer.is_none() { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - tracing::warn!( - "Rejected non-authoritative RBC-DAG shadow input from {:?}: {}", - peer, - error - ); - } - } - } - }) - }); - // Start bridge task that forwards reconstructed transaction data to core let bridge_task = decoded_rx.map(|mut decoded_rx| { let bridge_inner = inner.clone(); @@ -2552,8 +2189,6 @@ impl NetworkSyncer sf_event_task, rbc_event_task, rbc_service_task, - rbc_dag_shadow_event_task, - rbc_dag_shadow_service_task, cordial_knowledge_task, } } @@ -2599,11 +2234,6 @@ impl NetworkSyncer rbc_task.await.ok(); } let rbc_service_task = self.rbc_service_task; - if let Some(shadow_task) = self.rbc_dag_shadow_event_task { - shadow_task.abort(); - shadow_task.await.ok(); - } - let rbc_dag_shadow_service_task = self.rbc_dag_shadow_service_task; // Stop the cordial knowledge actor. self.cordial_knowledge_task.abort(); self.cordial_knowledge_task.await.ok(); @@ -2633,39 +2263,11 @@ impl NetworkSyncer // this FIFO barrier. Awaiting it keeps the runtime available to the // RBC actor while any earlier core action completes. let _ = inner.syncer.missing_parent_references().await; - let mut shadow_shutdown_timed_out = false; - if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { - match tokio::time::timeout(STARFISH_RBC_DAG_SHADOW_SHUTDOWN_TIMEOUT, shadow.shutdown()) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => tracing::warn!( - "Non-authoritative RBC-DAG shadow did not acknowledge shutdown: {error}" - ), - Err(_) => { - shadow_shutdown_timed_out = true; - tracing::warn!( - "Timed out stopping non-authoritative RBC-DAG shadow; detaching it from validator shutdown" - ); - if let Some(task) = rbc_dag_shadow_service_task.as_ref() { - task.abort(); - } - } - } - } let syncer = inner.syncer.stop(); if let Some(rbc_service_task) = rbc_service_task { rbc_service_task.abort(); rbc_service_task.await.ok(); } - if let Some(shadow_service_task) = rbc_dag_shadow_service_task { - match shadow_service_task.await { - Err(error) if !shadow_shutdown_timed_out => tracing::warn!( - "Non-authoritative RBC-DAG shadow supervisor failed during shutdown: {error}" - ), - _ => {} - } - } syncer } @@ -2777,7 +2379,6 @@ impl NetworkSyncer .await .ok()?; - let shadow_metrics = metrics.clone(); let mut handler = ConnectionHandler::new( &connection, universal_committer, @@ -2817,20 +2418,6 @@ impl NetworkSyncer ); } } - if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.peer_connected(peer_id) { - if shadow_transport_error_invalidates_comparison(&error) { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - tracing::warn!( - "Failed to notify RBC-DAG shadow that authority {} connected: {}", - peer_id, - error - ); - } - } if inner.dag_state.consensus_protocol.uses_bls() { for (round, signature) in inner.dag_state.precomputed_round_sigs() { @@ -2878,20 +2465,6 @@ impl NetworkSyncer ); } } - if let Some(ref shadow) = inner.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.peer_disconnected(peer_id) { - if shadow_transport_error_invalidates_comparison(&error) { - shadow_metrics - .starfish_rbc_dag_shadow_comparison_valid - .set(0); - } - tracing::warn!( - "Failed to notify RBC-DAG shadow that authority {} disconnected: {}", - peer_id, - error - ); - } - } inner.peer_senders.write().remove(&peer_id); inner.rbc_peer_senders.write().remove(&peer_id); if let Some(rbc_outbound_task) = rbc_outbound_task { diff --git a/crates/starfish-core/src/network.rs b/crates/starfish-core/src/network.rs index 6531bd77..5ee30976 100644 --- a/crates/starfish-core/src/network.rs +++ b/crates/starfish-core/src/network.rs @@ -83,24 +83,6 @@ pub struct ShardPayload { pub shard: ProvableShard, } -/// Non-authoritative Starfish-RBC-DAG carrier used by the persisted shadow -/// runtime. Both byte strings use the versioned canonical codecs from -/// `starfish_rbc_dag`; the network envelope deliberately adds no second -/// identity or authentication scheme. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct RbcDagShadowCarrier { - pub canonical_carrier: Vec, - pub authentication_sidecar: Vec, -} - -/// Content-only response for a phase-evidenced shadow carrier. Recovery can -/// satisfy READY/delivery, but it cannot grant optimistic admission or ECHO. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct RbcDagShadowCarrierResponse { - pub reference: BlockReference, - pub canonical_carrier: Vec, -} - /// A structured batch of block data, ordered by decreasing information density: /// full blocks first, then header-only blocks, then standalone shards. /// @@ -208,14 +190,6 @@ pub enum NetworkMessage { /// Starfish-RBC: return canonical header content. The receiver recomputes /// and checks its content-addressed reference before accepting it. RbcHeaderResponse(RbcCanonicalHeader), - /// Starfish-RBC-DAG milestone-three shadow carrier. This path is - /// observational and never feeds the authoritative DAG or consensus. - RbcDagShadowCarrier(RbcDagShadowCarrier), - /// Request canonical content after embedded phase evidence arrives before - /// the corresponding shadow carrier. - RbcDagShadowCarrierRequest(BlockReference), - /// Return content only; the receiver recomputes and checks the reference. - RbcDagShadowCarrierResponse(RbcDagShadowCarrierResponse), } impl NetworkMessage { @@ -243,9 +217,6 @@ impl NetworkMessage { }, Self::RbcHeaderRequest(_) => "rbc_header_request", Self::RbcHeaderResponse(_) => "rbc_header_response", - Self::RbcDagShadowCarrier(_) => "rbc_dag_shadow_carrier", - Self::RbcDagShadowCarrierRequest(_) => "rbc_dag_shadow_carrier_request", - Self::RbcDagShadowCarrierResponse(_) => "rbc_dag_shadow_carrier_response", } } } @@ -573,7 +544,7 @@ impl Worker { // Spawn the first task for handling pings let writer_clone = Arc::clone(&writer); let bytes_sent_total_clone = bytes_sent_total.clone(); - let ping_task = async move { + let ping_task = tokio::spawn(async move { let mut ping_deadline = start + PING_INTERVAL; loop { tokio::time::sleep_until(ping_deadline).await; @@ -593,12 +564,12 @@ impl Worker { } bytes_sent_total_clone.inc_by(12); // ping is 12-byte sized } - }; + }); // Spawn the second task for handling pong responses let writer_clone = Arc::clone(&writer); let bytes_sent_total_clone = bytes_sent_total.clone(); - let pong_task = async move { + let pong_task = tokio::spawn(async move { while let Some(ping) = pong_receiver.recv().await { if ping == 0 { tracing::warn!("Invalid ping: {ping}"); @@ -645,7 +616,7 @@ impl Worker { // Yield to ensure responsiveness tokio::task::yield_now().await; } - }; + }); // Spawn the third task(s) for handling message sending. // @@ -654,7 +625,7 @@ impl Worker { // backpressure encoding and starve catch-up for late joiners under // latency simulation. if connection_latency == 0.0 { - let message_task = async move { + let message_task = tokio::spawn(async move { while let Some(message) = receiver.recv().await { let request_type = message.request_type(); let serialized = bincode::serialize(&message).expect("Serialization failed"); @@ -690,10 +661,10 @@ impl Worker { } } } - }; + }); // Wait for all tasks to complete. - let _ = tokio::join!(ping_task, pong_task, message_task); + let _ = tokio::try_join!(ping_task, pong_task, message_task); return Ok(()); } @@ -701,7 +672,7 @@ impl Worker { // behavior by sleeping inside per-message tasks, but keep concurrency // bounded to avoid unbounded task buildup under heavy load. const MAX_IN_FLIGHT: usize = NETWORK_MESSAGE_CHANNEL_CAPACITY * 64; - let message_task = async move { + let message_task = tokio::spawn(async move { let mut join_set = tokio::task::JoinSet::new(); while let Some(message) = receiver.recv().await { @@ -760,10 +731,10 @@ impl Worker { tracing::error!("An inner task failed: {e:?}"); } } - }; + }); // Wait for all tasks to complete. - let _ = tokio::join!(ping_task, pong_task, message_task); + let _ = tokio::try_join!(ping_task, pong_task, message_task); Ok(()) } @@ -991,119 +962,12 @@ fn decode_ping(message: &[u8]) -> i64 { #[cfg(test)] mod tests { - use prometheus::Registry; - use super::*; use crate::{ - committee::Committee, crypto::{MacTag, TransactionsCommitment, dummy_signer}, starfish_rbc::{RbcInitialProof, RbcPhaseMessage}, }; - const NETWORK_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(6); - - async fn connected_pair( - addresses: &[SocketAddr; 2], - parameters: &NodeParameters, - ) -> (Network, Network, Connection, Connection) { - let committee = Committee::new_for_benchmarks(2); - let metrics_0 = Metrics::new(&Registry::new(), Some(&committee), None, None).0; - let metrics_1 = Metrics::new(&Registry::new(), Some(&committee), None, None).0; - let mut network_0 = - Network::from_socket_addresses(addresses, 0, addresses[0], metrics_0, parameters).await; - let mut network_1 = - Network::from_socket_addresses(addresses, 1, addresses[1], metrics_1, parameters).await; - - let (connection_0, connection_1) = tokio::time::timeout(NETWORK_LIFECYCLE_TIMEOUT, async { - tokio::join!( - network_0.connection_receiver().recv(), - network_1.connection_receiver().recv(), - ) - }) - .await - .expect("two-node network did not connect before the lifecycle timeout"); - let connection_0 = connection_0.expect("authority 0 connection channel closed"); - let connection_1 = connection_1.expect("authority 1 connection channel closed"); - assert_eq!(connection_0.peer_id, 1); - assert_eq!(connection_1.peer_id, 0); - (network_0, network_1, connection_0, connection_1) - } - - async fn assert_bidirectional_round_trip( - connection_0: &mut Connection, - connection_1: &mut Connection, - marker: RoundNumber, - ) { - connection_0 - .sender - .send(NetworkMessage::SubscribeBroadcastRequest(marker)) - .await - .unwrap(); - connection_1 - .sender - .send(NetworkMessage::SubscribeBroadcastRequest(marker + 1)) - .await - .unwrap(); - - let (received_by_0, received_by_1) = - tokio::time::timeout(NETWORK_LIFECYCLE_TIMEOUT, async { - tokio::join!(connection_0.receiver.recv(), connection_1.receiver.recv()) - }) - .await - .expect("two-node network did not exchange messages before the lifecycle timeout"); - assert!(matches!( - received_by_0, - Some(NetworkMessage::SubscribeBroadcastRequest(round)) if round == marker + 1 - )); - assert!(matches!( - received_by_1, - Some(NetworkMessage::SubscribeBroadcastRequest(round)) if round == marker - )); - } - - async fn abort_network_pair(network_0: Network, network_1: Network) { - // Match production shutdown: abort both listeners together. Awaiting - // the server tasks makes listener release deterministic for this test; - // dropping their worker senders must then cancel every scoped stream - // future and its OwnedWriteHalf. - network_0.abort_server(); - network_1.abort_server(); - let Network { - connection_receiver: connection_receiver_0, - server_task: server_task_0, - } = network_0; - let Network { - connection_receiver: connection_receiver_1, - server_task: server_task_1, - } = network_1; - drop(connection_receiver_0); - drop(connection_receiver_1); - let (result_0, result_1) = tokio::join!(server_task_0, server_task_1); - assert!(result_0.is_err_and(|error| error.is_cancelled())); - assert!(result_1.is_err_and(|error| error.is_cancelled())); - } - - async fn same_port_rebind_case(addresses: [SocketAddr; 2], latency_ms: Option) { - let parameters = NodeParameters { - mimic_latency: false, - uniform_latency_ms: latency_ms, - ..NodeParameters::default() - }; - - for cycle in 0..2 { - let (network_0, network_1, mut connection_0, mut connection_1) = - connected_pair(&addresses, ¶meters).await; - assert_bidirectional_round_trip(&mut connection_0, &mut connection_1, 10 + cycle).await; - - // NetworkSyncer drops its connection tasks before aborting the - // listener. Reproduce that ordering, then immediately construct - // the next cycle on the identical listener and active-bind ports. - drop(connection_0); - drop(connection_1); - abort_network_pair(network_0, network_1).await; - } - } - fn variant_index(message: &NetworkMessage) -> u32 { let bytes = bincode::serialize(message).unwrap(); u32::from_le_bytes(bytes[..4].try_into().unwrap()) @@ -1142,25 +1006,12 @@ mod tests { )); let request = NetworkMessage::RbcHeaderRequest(block_ref); let response = NetworkMessage::RbcHeaderResponse(header); - let shadow = NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { - canonical_carrier: vec![0xA3, 0xA4], - authentication_sidecar: vec![0xA5], - }); - let shadow_request = NetworkMessage::RbcDagShadowCarrierRequest(block_ref); - let shadow_response = - NetworkMessage::RbcDagShadowCarrierResponse(RbcDagShadowCarrierResponse { - reference: block_ref, - canonical_carrier: vec![0xA6, 0xA7], - }); for (message, expected_index, expected_kind) in [ (initial, 11, "rbc_initial"), (phase, 12, "rbc_ready"), (request, 13, "rbc_header_request"), (response, 14, "rbc_header_response"), - (shadow, 15, "rbc_dag_shadow_carrier"), - (shadow_request, 16, "rbc_dag_shadow_carrier_request"), - (shadow_response, 17, "rbc_dag_shadow_carrier_response"), ] { assert_eq!(variant_index(&message), expected_index); assert_eq!(message.request_type(), expected_kind); @@ -1170,29 +1021,4 @@ mod tests { assert_eq!(variant_index(&decoded), expected_index); } } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn scoped_connection_tasks_allow_immediate_same_port_rebind() { - // Active sockets bind to listener_port * 10. Keep those derived ports - // below the usual Linux ephemeral range to avoid unrelated allocation - // races while staying above the repository's validator-test fixtures. - same_port_rebind_case( - [ - SocketAddr::from(([127, 0, 0, 1], 3_200)), - SocketAddr::from(([127, 0, 0, 1], 3_201)), - ], - None, - ) - .await; - // Any nonzero configured latency selects the JoinSet-backed writer - // branch, so this also covers cancellation of its in-flight tasks. - same_port_rebind_case( - [ - SocketAddr::from(([127, 0, 0, 1], 3_220)), - SocketAddr::from(([127, 0, 0, 1], 3_221)), - ], - Some(5.0), - ) - .await; - } } diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs index 24812f40..fa7287a6 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/journal.rs @@ -8,7 +8,7 @@ //! decoding. Callers validate canonical bytes before journaling them; the //! reducer pins the exact byte strings and rejects any later alternative. -use std::{collections::BTreeMap, error::Error, fmt, sync::Arc}; +use std::{collections::BTreeMap, error::Error, fmt}; use crate::types::{AuthorityIndex, BlockReference, RoundNumber}; @@ -1010,10 +1010,6 @@ pub enum JournalErrorV1 { outer: BlockReference, index: usize, }, - StaleValidatedBatch { - expected_events: usize, - actual_events: usize, - }, } impl fmt::Display for JournalErrorV1 { @@ -1030,17 +1026,7 @@ pub struct WriteAheadJournalV1 { context: RbcDagContextV1, own_authority: AuthorityIndex, durable_events: Vec, - snapshot: Arc, -} - -/// A transition checked against one exact journal prefix. Keeping only the -/// newly appended events avoids cloning the complete durable history on every -/// live shadow input. -pub(crate) struct ValidatedJournalBatchV1 { - base_event_count: usize, - base_snapshot: Arc, - events: Vec, - snapshot: Arc, + snapshot: JournalSnapshotV1, } impl WriteAheadJournalV1 { @@ -1049,7 +1035,7 @@ impl WriteAheadJournalV1 { context, own_authority, durable_events: Vec::new(), - snapshot: Arc::new(JournalSnapshotV1::new(context, own_authority)), + snapshot: JournalSnapshotV1::new(context, own_authority), } } @@ -1057,43 +1043,10 @@ impl WriteAheadJournalV1 { /// durable and visible. A real backend maps this boundary to its durable /// transaction commit. pub fn append(&mut self, event: JournalEventV1) -> Result<(), JournalErrorV1> { - let mut next = self.snapshot.as_ref().clone(); + let mut next = self.snapshot.clone(); next.apply(&event)?; self.durable_events.push(event); - self.snapshot = Arc::new(next); - Ok(()) - } - - pub(crate) fn validate_batch( - &self, - events: Vec, - ) -> Result { - let mut snapshot = self.snapshot.as_ref().clone(); - for event in &events { - snapshot.apply(event)?; - } - Ok(ValidatedJournalBatchV1 { - base_event_count: self.durable_events.len(), - base_snapshot: Arc::clone(&self.snapshot), - events, - snapshot: Arc::new(snapshot), - }) - } - - pub(crate) fn commit_validated_batch( - &mut self, - batch: ValidatedJournalBatchV1, - ) -> Result<(), JournalErrorV1> { - if self.durable_events.len() != batch.base_event_count - || !Arc::ptr_eq(&self.snapshot, &batch.base_snapshot) - { - return Err(JournalErrorV1::StaleValidatedBatch { - expected_events: batch.base_event_count, - actual_events: self.durable_events.len(), - }); - } - self.durable_events.extend(batch.events); - self.snapshot = batch.snapshot; + self.snapshot = next; Ok(()) } @@ -1117,7 +1070,7 @@ impl WriteAheadJournalV1 { } pub fn snapshot(&self) -> &JournalSnapshotV1 { - self.snapshot.as_ref() + &self.snapshot } /// Rebuild volatile state in exact durable order. @@ -1142,7 +1095,7 @@ impl WriteAheadJournalV1 { context, own_authority, durable_events, - snapshot: Arc::new(snapshot), + snapshot, }) } } @@ -1418,121 +1371,6 @@ mod tests { assert!(assertion(after.snapshot())); } - #[test] - fn validated_batch_matches_sequential_append_and_restart() { - let mut batched = journal(); - let mut sequential = batched.clone(); - let own_candidate = candidate(1, 1, 0x0A, Vec::new(), None); - let events = vec![ - outbound_content_event(&batched, &own_candidate), - fix_event(&batched, own_candidate.reference()), - ]; - - let batch = batched.validate_batch(events.clone()).unwrap(); - batched.commit_validated_batch(batch).unwrap(); - for event in events { - sequential.append(event).unwrap(); - } - - assert_eq!(batched.durable_events(), sequential.durable_events()); - assert_eq!(batched.snapshot(), sequential.snapshot()); - assert_eq!(batched.restart().unwrap(), sequential.restart().unwrap()); - } - - #[test] - fn failed_batch_validation_leaves_the_journal_unchanged() { - let journal = journal(); - let before = journal.clone(); - let own = candidate(1, 1, 0x0B, Vec::new(), None).reference(); - - let result = journal.validate_batch(vec![fix_event(&journal, own)]); - - assert!(matches!( - result, - Err(JournalErrorV1::OutboundContentNotPersisted(reference)) if reference == own - )); - assert_eq!(journal, before); - } - - #[test] - fn validated_batch_rejects_an_intervening_append_without_mutation() { - let mut journal = journal(); - let planned = candidate(0, 1, 0x0C, Vec::new(), None); - let intervening = candidate(2, 1, 0x0D, Vec::new(), None); - let batch = journal - .validate_batch(vec![JournalEventV1::RetainCandidateContent { - context: journal.context, - candidate: planned, - }]) - .unwrap(); - journal - .append(JournalEventV1::RetainCandidateContent { - context: journal.context, - candidate: intervening, - }) - .unwrap(); - let after_intervening = journal.clone(); - - assert_eq!( - journal.commit_validated_batch(batch).unwrap_err(), - JournalErrorV1::StaleValidatedBatch { - expected_events: 0, - actual_events: 1, - } - ); - assert_eq!(journal, after_intervening); - assert_eq!( - journal.restart().unwrap(), - after_intervening.restart().unwrap() - ); - } - - #[test] - fn validated_batch_rejects_a_divergent_equal_length_journal() { - let mut source = journal(); - let mut divergent = source.clone(); - let source_candidate = candidate(0, 1, 0x0E, Vec::new(), None); - let divergent_candidate = candidate(2, 1, 0x0F, Vec::new(), None); - source - .append(JournalEventV1::RetainCandidateContent { - context: source.context, - candidate: source_candidate.clone(), - }) - .unwrap(); - divergent - .append(JournalEventV1::RetainCandidateContent { - context: divergent.context, - candidate: divergent_candidate, - }) - .unwrap(); - assert_eq!( - source.durable_events().len(), - divergent.durable_events().len() - ); - assert_ne!(source.snapshot(), divergent.snapshot()); - - let batch = source - .validate_batch(vec![JournalEventV1::LockReady { - context: source.context, - target: source_candidate.reference(), - }]) - .unwrap(); - let before_commit = divergent.clone(); - - assert_eq!( - divergent.commit_validated_batch(batch).unwrap_err(), - JournalErrorV1::StaleValidatedBatch { - expected_events: 1, - actual_events: 1, - } - ); - assert_eq!(divergent, before_commit); - assert_eq!( - divergent.restart().unwrap(), - before_commit.restart().unwrap() - ); - } - #[test] fn authenticated_ingress_sequence_and_bytes_survive_restart_in_order() { let mut journal = journal(); diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs index 31f82af0..2a3afd17 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/mod.rs @@ -4,14 +4,12 @@ //! Canonical carrier types for the experimental embedded-RBC Starfish DAG. //! //! This module is deliberately independent from the implemented direct-message -//! `starfish_rbc` protocol. An opt-in persisted shadow adapter consumes these -//! types without influencing consensus; authoritative integration remains a -//! later milestone. +//! `starfish_rbc` protocol. Runtime and consensus integration are later +//! milestones. pub mod journal; pub mod model; pub mod projection; -pub mod storage; use std::{ collections::{BTreeSet, HashSet}, @@ -69,11 +67,6 @@ const CARRIER_AUTHENTICATION_KIND: u8 = 0; const AUTHENTICATION_BASE_SIZE: usize = 123; const AUTHENTICATION_MAC_SIZE: usize = AUTHENTICATION_BASE_SIZE + 2; -#[cfg(test)] -std::thread_local! { - static COMMITTEE_ID_DERIVATIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum RbcPhaseStatementV1 { Echo { target: BlockReference }, @@ -410,59 +403,19 @@ pub struct CandidateCarrierV1 { } impl CandidateCarrierV1 { - /// Convenience constructor for tests and one-shot callers. - /// - /// Runtime code should build one [`RbcDagCommitteeContextV1`] and call - /// [`Self::try_new_with_committee`] so the committee's complete public-key - /// transcript is not rehashed for every carrier. pub fn try_new(args: CarrierHeaderV1Args, committee: &Committee) -> Result { - Self::try_from_header_internal(CarrierHeaderV1::from_args(args), committee, None, None) - } - - pub fn try_new_with_committee( - args: CarrierHeaderV1Args, - committee: &RbcDagCommitteeContextV1, - ) -> Result { - Self::try_from_header_internal( - CarrierHeaderV1::from_args(args), - committee.committee(), - Some(committee.committee_id()), - None, - ) + Self::try_from_header(CarrierHeaderV1::from_args(args), committee, None) } - /// Convenience constructor for tests and one-shot callers. Runtime code - /// should prefer [`Self::try_from_header_with_committee`]. pub fn try_from_header( - header: CarrierHeaderV1, - committee: &Committee, - expected_reference: Option, - ) -> Result { - Self::try_from_header_internal(header, committee, None, expected_reference) - } - - pub fn try_from_header_with_committee( - header: CarrierHeaderV1, - committee: &RbcDagCommitteeContextV1, - expected_reference: Option, - ) -> Result { - Self::try_from_header_internal( - header, - committee.committee(), - Some(committee.committee_id()), - expected_reference, - ) - } - - fn try_from_header_internal( mut header: CarrierHeaderV1, committee: &Committee, - cached_committee_id: Option, expected_reference: Option, ) -> Result { normalize_acknowledgments(&mut header)?; - validate_outer_header(&header, committee, cached_committee_id.is_some())?; + validate_outer_header(&header, committee)?; let reference = carrier_reference(&header)?; + let committee_id = RbcDagCommitteeId::derive(committee)?; if let Some(expected) = expected_reference { if expected != reference { return Err(RbcDagError::ReferenceMismatch { @@ -471,10 +424,6 @@ impl CandidateCarrierV1 { }); } } - let committee_id = match cached_committee_id { - Some(committee_id) => committee_id, - None => RbcDagCommitteeId::derive(committee)?, - }; Ok(Self { header: Arc::new(header), reference, @@ -488,26 +437,7 @@ impl CandidateCarrierV1 { expected_reference: Option, ) -> Result { let header = decode_header(bytes, AckEncoding::Expanded)?; - let candidate = - Self::try_from_header_internal(header, committee, None, expected_reference)?; - if candidate.canonical_content_bytes()?.as_slice() != bytes { - return Err(RbcDagError::NonCanonicalAcknowledgments); - } - Ok(candidate) - } - - pub fn decode_content_with_committee( - bytes: &[u8], - committee: &RbcDagCommitteeContextV1, - expected_reference: Option, - ) -> Result { - let header = decode_header(bytes, AckEncoding::Expanded)?; - let candidate = Self::try_from_header_internal( - header, - committee.committee(), - Some(committee.committee_id()), - expected_reference, - )?; + let candidate = Self::try_from_header(header, committee, expected_reference)?; if candidate.canonical_content_bytes()?.as_slice() != bytes { return Err(RbcDagError::NonCanonicalAcknowledgments); } @@ -520,21 +450,7 @@ impl CandidateCarrierV1 { expected_reference: Option, ) -> Result { let header = decode_header(bytes, AckEncoding::Compressed)?; - Self::try_from_header_internal(header, committee, None, expected_reference) - } - - pub fn decode_wire_with_committee( - bytes: &[u8], - committee: &RbcDagCommitteeContextV1, - expected_reference: Option, - ) -> Result { - let header = decode_header(bytes, AckEncoding::Compressed)?; - Self::try_from_header_internal( - header, - committee.committee(), - Some(committee.committee_id()), - expected_reference, - ) + Self::try_from_header(header, committee, expected_reference) } pub fn header(&self) -> &CarrierHeaderV1 { @@ -563,24 +479,6 @@ impl CandidateCarrierV1 { ) -> Result, RbcDagProjectionError> { let committee_id = RbcDagCommitteeId::derive(committee) .map_err(|_| RbcDagProjectionError::CommitteeMismatch)?; - self.validate_consensus_vertex_with_validated_committee(committee, committee_id) - } - - pub fn validate_consensus_vertex_with_committee( - &self, - committee: &RbcDagCommitteeContextV1, - ) -> Result, RbcDagProjectionError> { - self.validate_consensus_vertex_with_validated_committee( - committee.committee(), - committee.committee_id(), - ) - } - - fn validate_consensus_vertex_with_validated_committee( - &self, - committee: &Committee, - committee_id: RbcDagCommitteeId, - ) -> Result, RbcDagProjectionError> { if committee_id != self.committee_id { return Err(RbcDagProjectionError::CommitteeMismatch); } @@ -689,24 +587,8 @@ impl CarrierAuthenticationV1 { bytes } - /// Convenience decoder for one-shot callers. Runtime code should prefer - /// [`Self::decode_wire_with_committee`]. pub fn decode_wire(bytes: &[u8], committee: &Committee) -> Result { validate_committee(committee)?; - Self::decode_wire_with_validated_committee(bytes, committee) - } - - pub fn decode_wire_with_committee( - bytes: &[u8], - committee: &RbcDagCommitteeContextV1, - ) -> Result { - Self::decode_wire_with_validated_committee(bytes, committee.committee()) - } - - fn decode_wire_with_validated_committee( - bytes: &[u8], - committee: &Committee, - ) -> Result { let mut decoder = Decoder::new(bytes); decoder.expect_marker(CONTENT_FORMAT_FIELD)?; let version = decoder.read_u8()?; @@ -805,8 +687,6 @@ pub struct RbcDagCommitteeId([u8; COMMITTEE_ID_SIZE]); impl RbcDagCommitteeId { pub fn derive(committee: &Committee) -> Result { validate_committee(committee)?; - #[cfg(test)] - COMMITTEE_ID_DERIVATIONS.with(|count| count.set(count.get().saturating_add(1))); let committee_size = u16::try_from(committee.len()) .map_err(|_| RbcDagError::InvalidCommittee("committee too large"))?; let info_length = u16::try_from(committee.info_length()) @@ -856,51 +736,6 @@ impl fmt::Debug for RbcDagCommitteeId { } } -/// Validated, reusable committee capability for the Starfish-RBC-DAG hot -/// path. -/// -/// Construction validates the committee and hashes its complete key -/// transcript exactly once. Candidate decoding, authentication, and -/// projection APIs that accept this capability perform only constant-time ID -/// comparisons before using the retained committee. -#[derive(Clone)] -pub struct RbcDagCommitteeContextV1 { - committee: Arc, - committee_id: RbcDagCommitteeId, -} - -impl RbcDagCommitteeContextV1 { - pub fn new(committee: Arc) -> Result { - let committee_id = RbcDagCommitteeId::derive(&committee)?; - Ok(Self { - committee, - committee_id, - }) - } - - pub fn committee(&self) -> &Committee { - &self.committee - } - - pub fn committee_arc(&self) -> Arc { - Arc::clone(&self.committee) - } - - pub fn committee_id(&self) -> RbcDagCommitteeId { - self.committee_id - } -} - -impl fmt::Debug for RbcDagCommitteeContextV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RbcDagCommitteeContextV1") - .field("committee_id", &self.committee_id) - .field("committee_size", &self.committee.len()) - .finish() - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RbcDagContextV1 { protocol_instance: RbcDagProtocolInstanceId, @@ -909,8 +744,6 @@ pub struct RbcDagContextV1 { } impl RbcDagContextV1 { - /// Convenience constructor for one-shot callers. Runtime code should - /// prefer [`Self::new_with_committee`]. pub fn new( protocol_instance: RbcDagProtocolInstanceId, committee: &Committee, @@ -923,18 +756,6 @@ impl RbcDagContextV1 { }) } - pub fn new_with_committee( - protocol_instance: RbcDagProtocolInstanceId, - committee: &RbcDagCommitteeContextV1, - authentication_scheme: BlockAuthenticationScheme, - ) -> Self { - Self { - protocol_instance, - committee_id: committee.committee_id(), - authentication_scheme, - } - } - pub fn protocol_instance(&self) -> RbcDagProtocolInstanceId { self.protocol_instance } @@ -947,40 +768,13 @@ impl RbcDagContextV1 { self.authentication_scheme } - /// Convenience authorizer for one-shot callers. Runtime code should - /// prefer [`Self::authenticate_with_committee`]. pub fn authenticate( &self, candidate: &CandidateCarrierV1, committee: &Committee, authorizer: CarrierAuthorizerV1<'_>, ) -> Result { - let committee_id = RbcDagCommitteeId::derive(committee)?; - self.authenticate_with_validated_committee(candidate, committee, committee_id, authorizer) - } - - pub fn authenticate_with_committee( - &self, - candidate: &CandidateCarrierV1, - committee: &RbcDagCommitteeContextV1, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - self.authenticate_with_validated_committee( - candidate, - committee.committee(), - committee.committee_id(), - authorizer, - ) - } - - fn authenticate_with_validated_committee( - &self, - candidate: &CandidateCarrierV1, - committee: &Committee, - committee_id: RbcDagCommitteeId, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - self.ensure_committee_id(committee_id)?; + self.ensure_committee(committee)?; self.ensure_candidate(candidate)?; if authorizer.scheme() != self.authentication_scheme { return Err(RbcDagError::AuthenticationSchemeMismatch); @@ -1056,9 +850,6 @@ impl RbcDagContextV1 { /// The returned capability has private fields so persistence and network /// adapters cannot substitute a freely constructed, same-scheme sidecar /// for the one produced by the configured authorizer. - /// - /// Convenience local authorizer for one-shot callers. Runtime code should - /// prefer [`Self::authenticate_local_with_committee`]. pub fn authenticate_local( &self, candidate: CandidateCarrierV1, @@ -1073,175 +864,6 @@ impl RbcDagContextV1 { }) } - pub fn authenticate_local_with_committee( - &self, - candidate: CandidateCarrierV1, - committee: &RbcDagCommitteeContextV1, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - let authentication = self.authenticate_with_committee(&candidate, committee, authorizer)?; - Ok(LocallyAuthenticatedCarrierV1 { - candidate, - authentication, - context: *self, - }) - } - - /// Recover the opaque local-authentication capability from an exact - /// persisted sidecar without regenerating it. - /// - /// Signature modes verify the persisted public proof and the configured - /// local signer's public key. MAC mode verifies every ordered vector entry - /// with the configured outbound keyring; checking only this node's entry - /// would not prove that the locally exposed full vector was generated - /// correctly. - /// - /// Convenience recovery verifier for one-shot callers. Runtime code - /// should prefer [`Self::verify_local_authentication_with_committee`]. - pub fn verify_local_authentication( - &self, - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - committee: &Committee, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - let committee_id = RbcDagCommitteeId::derive(committee)?; - self.verify_local_authentication_with_validated_committee( - candidate, - authentication, - committee, - committee_id, - authorizer, - ) - } - - pub fn verify_local_authentication_with_committee( - &self, - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - committee: &RbcDagCommitteeContextV1, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - self.verify_local_authentication_with_validated_committee( - candidate, - authentication, - committee.committee(), - committee.committee_id(), - authorizer, - ) - } - - fn verify_local_authentication_with_validated_committee( - &self, - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - committee: &Committee, - committee_id: RbcDagCommitteeId, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - self.ensure_committee_id(committee_id)?; - self.ensure_candidate(&candidate)?; - if authentication.scheme() != self.authentication_scheme - || authorizer.scheme() != self.authentication_scheme - { - return Err(RbcDagError::AuthenticationSchemeMismatch); - } - let reference = candidate.reference; - if authorizer.authority() != reference.authority { - return Err(RbcDagError::AuthorizerAuthorityMismatch { - expected: reference.authority, - actual: authorizer.authority(), - }); - } - - match (authorizer, &authentication) { - ( - CarrierAuthorizerV1::Ed25519 { signer, .. }, - CarrierAuthenticationV1::Ed25519(signature), - ) => { - let expected = committee - .get_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - if &signer.public_key() != expected { - return Err(RbcDagError::AuthorizerKeyMismatch); - } - expected - .verify_digest_signature( - &self.public_authentication_digest(reference), - signature, - ) - .map_err(|_| RbcDagError::InvalidAuthentication)?; - } - ( - CarrierAuthorizerV1::MlDsa44 { signer, .. }, - CarrierAuthenticationV1::MlDsa44(signature), - ) => { - let expected = committee - .get_ml_dsa_44_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - if &signer.public_key() != expected { - return Err(RbcDagError::AuthorizerKeyMismatch); - } - let digest = BlockDigest::from(self.public_authentication_digest(reference)); - expected - .verify_digest_signature(&digest, signature) - .map_err(|_| RbcDagError::InvalidAuthentication)?; - } - ( - CarrierAuthorizerV1::MlDsa65 { signer, .. }, - CarrierAuthenticationV1::MlDsa65(signature), - ) => { - let expected = committee - .get_ml_dsa_65_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - if &signer.public_key() != expected { - return Err(RbcDagError::AuthorizerKeyMismatch); - } - let digest = BlockDigest::from(self.public_authentication_digest(reference)); - expected - .verify_digest_signature(&digest, signature) - .map_err(|_| RbcDagError::InvalidAuthentication)?; - } - ( - CarrierAuthorizerV1::MacVector { keys, .. }, - CarrierAuthenticationV1::MacVector(vector), - ) => { - let expected_length = committee.len() * MAC_TAG_SIZE; - if vector.as_bytes().len() != expected_length { - return Err(RbcDagError::InvalidMacVectorLength { - expected: expected_length, - actual: vector.as_bytes().len(), - }); - } - if keys.len() != committee.len() { - return Err(RbcDagError::InvalidKeyringLength { - expected: committee.len(), - actual: keys.len(), - }); - } - for recipient in committee.authorities() { - let expected = keys[recipient as usize] - .compute_rbc_tag(&self.mac_authentication_statement(reference, recipient)); - let actual = vector - .tag(recipient) - .ok_or(RbcDagError::InvalidAuthentication)?; - if actual != expected { - return Err(RbcDagError::InvalidAuthentication); - } - } - } - _ => return Err(RbcDagError::AuthenticationSchemeMismatch), - } - - Ok(LocallyAuthenticatedCarrierV1 { - candidate, - authentication, - context: *self, - }) - } - - /// Convenience inbound verifier for one-shot callers. Runtime code should - /// prefer [`Self::verify_authentication_with_committee`]. pub fn verify_authentication( &self, candidate: CandidateCarrierV1, @@ -1250,45 +872,7 @@ impl RbcDagContextV1 { committee: &Committee, mac_keys: &[MacKey], ) -> Result { - let committee_id = RbcDagCommitteeId::derive(committee)?; - self.verify_authentication_with_validated_committee( - candidate, - authentication, - receiver, - committee, - committee_id, - mac_keys, - ) - } - - pub fn verify_authentication_with_committee( - &self, - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - receiver: AuthorityIndex, - committee: &RbcDagCommitteeContextV1, - mac_keys: &[MacKey], - ) -> Result { - self.verify_authentication_with_validated_committee( - candidate, - authentication, - receiver, - committee.committee(), - committee.committee_id(), - mac_keys, - ) - } - - fn verify_authentication_with_validated_committee( - &self, - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - receiver: AuthorityIndex, - committee: &Committee, - committee_id: RbcDagCommitteeId, - mac_keys: &[MacKey], - ) -> Result { - self.ensure_committee_id(committee_id)?; + self.ensure_committee(committee)?; self.ensure_candidate(&candidate)?; if !committee.known_authority(receiver) { return Err(RbcDagError::UnknownAuthority(receiver)); @@ -1385,7 +969,8 @@ impl RbcDagContextV1 { blake3::hash(&self.public_authentication_statement(reference)).into() } - fn ensure_committee_id(&self, actual: RbcDagCommitteeId) -> Result<(), RbcDagError> { + fn ensure_committee(&self, committee: &Committee) -> Result<(), RbcDagError> { + let actual = RbcDagCommitteeId::derive(committee)?; if actual != self.committee_id { return Err(RbcDagError::CommitteeIdMismatch); } @@ -1584,11 +1169,8 @@ impl Error for RbcDagError {} fn validate_outer_header( header: &CarrierHeaderV1, committee: &Committee, - committee_is_validated: bool, ) -> Result<(), RbcDagError> { - if !committee_is_validated { - validate_committee(committee)?; - } + validate_committee(committee)?; if header.carrier_round == 0 { return Err(RbcDagError::GenesisCarrier); } @@ -2896,361 +2478,6 @@ mod tests { )); } - #[test] - fn cached_committee_context_hashes_the_key_transcript_once_across_hot_paths() { - COMMITTEE_ID_DERIVATIONS.with(|count| count.set(0)); - - let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - assert_eq!(COMMITTEE_ID_DERIVATIONS.with(std::cell::Cell::get), 1); - - let candidate = - CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) - .unwrap(); - let content = candidate.canonical_content_bytes().unwrap(); - let wire = candidate.canonical_wire_bytes().unwrap(); - let decoded_content = CandidateCarrierV1::decode_content_with_committee( - &content, - &committee_context, - Some(candidate.reference()), - ) - .unwrap(); - let decoded_wire = CandidateCarrierV1::decode_wire_with_committee( - &wire, - &committee_context, - Some(candidate.reference()), - ) - .unwrap(); - assert_eq!(decoded_content, candidate); - assert_eq!(decoded_wire, candidate); - - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(), - &committee_context, - BlockAuthenticationScheme::MacVector, - ); - let keyrings = mac_keyrings_for_test(committee.len()); - let authentication = context - .authenticate_with_committee( - &candidate, - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: candidate.header().author(), - keys: &keyrings[candidate.header().author() as usize], - }, - ) - .unwrap(); - let authentication_wire = authentication.canonical_wire_bytes(); - let decoded_authentication = CarrierAuthenticationV1::decode_wire_with_committee( - &authentication_wire, - &committee_context, - ) - .unwrap(); - context - .verify_authentication_with_committee( - candidate.clone(), - decoded_authentication, - 1, - &committee_context, - &keyrings[1], - ) - .unwrap(); - context - .verify_local_authentication_with_committee( - candidate.clone(), - authentication, - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: candidate.header().author(), - keys: &keyrings[candidate.header().author() as usize], - }, - ) - .unwrap(); - context - .authenticate_local_with_committee( - candidate.clone(), - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: candidate.header().author(), - keys: &keyrings[candidate.header().author() as usize], - }, - ) - .unwrap(); - candidate - .validate_consensus_vertex_with_committee(&committee_context) - .unwrap(); - - let mut projection = - projection::CertifiedProjectionModel::from_committee_context(committee_context.clone()); - projection.stage_carrier(candidate.clone()).unwrap(); - assert!(matches!( - projection.try_project(candidate.reference()), - Err(projection::CertifiedProjectionError::CarrierNotDelivered(reference)) - if reference == candidate.reference() - )); - - assert_eq!(COMMITTEE_ID_DERIVATIONS.with(std::cell::Cell::get), 1); - } - - #[test] - fn cached_committee_context_rejects_cross_committee_hot_path_use() { - let committee = Committee::new_test(vec![1; 4]); - let other_committee = Committee::new_test(vec![1, 1, 1, 2]); - let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let other_context = RbcDagCommitteeContextV1::new(Arc::clone(&other_committee)).unwrap(); - let candidate = - CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) - .unwrap(); - let protocol_context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xB7; 32]).unwrap(), - &committee_context, - BlockAuthenticationScheme::MacVector, - ); - let keyrings = mac_keyrings_for_test(committee.len()); - let authentication = protocol_context - .authenticate_with_committee( - &candidate, - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: candidate.header().author(), - keys: &keyrings[candidate.header().author() as usize], - }, - ) - .unwrap(); - - assert!(matches!( - protocol_context.authenticate_with_committee( - &candidate, - &other_context, - CarrierAuthorizerV1::MacVector { - authority: candidate.header().author(), - keys: &keyrings[candidate.header().author() as usize], - }, - ), - Err(RbcDagError::CommitteeIdMismatch) - )); - assert!(matches!( - protocol_context.verify_authentication_with_committee( - candidate.clone(), - authentication.clone(), - 1, - &other_context, - &keyrings[1], - ), - Err(RbcDagError::CommitteeIdMismatch) - )); - assert!(matches!( - protocol_context.verify_local_authentication_with_committee( - candidate.clone(), - authentication, - &other_context, - CarrierAuthorizerV1::MacVector { - authority: candidate.header().author(), - keys: &keyrings[candidate.header().author() as usize], - }, - ), - Err(RbcDagError::CommitteeIdMismatch) - )); - assert!(matches!( - candidate.validate_consensus_vertex_with_committee(&other_context), - Err(RbcDagProjectionError::CommitteeMismatch) - )); - - let mut projection = - projection::CertifiedProjectionModel::from_committee_context(other_context); - assert_eq!( - projection.stage_carrier(candidate), - Err(projection::CertifiedProjectionError::CommitteeMismatch) - ); - } - - #[test] - fn persisted_ml_dsa_sidecar_recovers_exact_local_capability_and_rejects_tampering() { - let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let candidate = - CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) - .unwrap(); - let instance = RbcDagProtocolInstanceId::new([0xC8; 32]).unwrap(); - let context = RbcDagContextV1::new_with_committee( - instance, - &committee_context, - BlockAuthenticationScheme::MlDsa65, - ); - let signer = dummy_ml_dsa_65_signer(); - let authentication = context - .authenticate_with_committee( - &candidate, - &committee_context, - CarrierAuthorizerV1::MlDsa65 { - authority: candidate.header().author(), - signer: &signer, - }, - ) - .unwrap(); - let persisted_wire = authentication.canonical_wire_bytes(); - let persisted_authentication = CarrierAuthenticationV1::decode_wire_with_committee( - &persisted_wire, - &committee_context, - ) - .unwrap(); - let recovered = context - .verify_local_authentication_with_committee( - candidate.clone(), - persisted_authentication, - &committee_context, - CarrierAuthorizerV1::MlDsa65 { - authority: candidate.header().author(), - signer: &signer, - }, - ) - .unwrap(); - assert_eq!(recovered.authentication(), &authentication); - assert_eq!( - recovered.authentication().canonical_wire_bytes(), - persisted_wire - ); - - let CarrierAuthenticationV1::MlDsa65(signature) = &authentication else { - unreachable!() - }; - let mut tampered_bytes = [0; ML_DSA_65_SIGNATURE_SIZE]; - tampered_bytes.copy_from_slice(signature.as_ref()); - tampered_bytes[0] ^= 1; - let tampered = - CarrierAuthenticationV1::MlDsa65(MlDsa65SignatureBytes::from_bytes(tampered_bytes)); - assert!(matches!( - context.verify_local_authentication_with_committee( - candidate.clone(), - tampered, - &committee_context, - CarrierAuthorizerV1::MlDsa65 { - authority: candidate.header().author(), - signer: &signer, - }, - ), - Err(RbcDagError::InvalidAuthentication) - )); - assert!(matches!( - context.verify_local_authentication_with_committee( - candidate.clone(), - authentication.clone(), - &committee_context, - CarrierAuthorizerV1::MlDsa65 { - authority: 2, - signer: &signer, - }, - ), - Err(RbcDagError::AuthorizerAuthorityMismatch { - expected: 3, - actual: 2, - }) - )); - - let wrong_context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xC9; 32]).unwrap(), - &committee_context, - BlockAuthenticationScheme::MlDsa65, - ); - assert!(matches!( - wrong_context.verify_local_authentication_with_committee( - candidate, - authentication, - &committee_context, - CarrierAuthorizerV1::MlDsa65 { - authority: 3, - signer: &signer, - }, - ), - Err(RbcDagError::InvalidAuthentication) - )); - } - - #[test] - fn persisted_local_mac_recovery_verifies_every_vector_entry_and_length() { - let committee = Committee::new_test(vec![1; 4]); - let committee_context = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let candidate = - CandidateCarrierV1::try_new_with_committee(full_args(&committee), &committee_context) - .unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xD9; 32]).unwrap(), - &committee_context, - BlockAuthenticationScheme::MacVector, - ); - let keyrings = mac_keyrings_for_test(committee.len()); - let author = candidate.header().author() as usize; - let authentication = context - .authenticate_with_committee( - &candidate, - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ) - .unwrap(); - context - .verify_local_authentication_with_committee( - candidate.clone(), - authentication.clone(), - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ) - .unwrap(); - - let CarrierAuthenticationV1::MacVector(vector) = &authentication else { - unreachable!() - }; - let mut poisoned_bytes = vector.as_bytes().to_vec(); - poisoned_bytes[2 * MAC_TAG_SIZE] ^= 1; - let poisoned = - CarrierAuthenticationV1::MacVector(FlatMacVector::from_bytes(poisoned_bytes).unwrap()); - context - .verify_authentication_with_committee( - candidate.clone(), - poisoned.clone(), - 1, - &committee_context, - &keyrings[1], - ) - .expect("a different recipient's entry remains valid"); - assert!(matches!( - context.verify_local_authentication_with_committee( - candidate.clone(), - poisoned, - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ), - Err(RbcDagError::InvalidAuthentication) - )); - - let short = CarrierAuthenticationV1::MacVector( - FlatMacVector::from_bytes( - vector.as_bytes()[..vector.as_bytes().len() - MAC_TAG_SIZE].to_vec(), - ) - .unwrap(), - ); - assert!(matches!( - context.verify_local_authentication_with_committee( - candidate, - short, - &committee_context, - CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ), - Err(RbcDagError::InvalidMacVectorLength { .. }) - )); - } - #[test] fn sidecar_wire_has_frozen_flat_mac_shape() { let committee = Committee::new_test(vec![1; 4]); diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs index 192385aa..f1506bad 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/model.rs @@ -19,7 +19,6 @@ use std::{ use crate::{ committee::Committee, - crypto::Blake3Hasher, types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, }; @@ -35,9 +34,6 @@ use super::{ pub const EXECUTABLE_MODEL_ADMISSION_WINDOW_V1: RoundNumber = 2; pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 4; -const MODEL_LINEAGE_DERIVE_CONTEXT: &str = "starfish-rbc-dag-model-lineage-v1"; -type ModelLineage = [u8; 32]; - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum IngressAuthentication { Authenticated, @@ -63,116 +59,6 @@ pub enum ModelEffect { CarrierRoundAdvanced(RoundNumber), } -/// One proof-critical or externally observable step of a reducer transition. -/// -/// The order is part of the runtime contract. A caller may plan a transition -/// on a clone, persist these entries in order, and only then install the -/// planned model with [`RbcDagModel::commit_plan`]. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ModelTraceEvent { - /// The first authenticated value selected for a remote carrier slot. - AdmissionLocked(BlockReference), - /// A locally generated ECHO or READY became slot-global and immutable. - LocalPhaseLocked(RbcPhaseStatementV1), - /// One exact entry of an enclosing carrier's authenticated phase log is - /// about to be applied. Any lock enabled by that entry follows this event. - PhaseBatchEntryApplied { - outer: BlockReference, - index: usize, - sender: AuthorityIndex, - statement: RbcPhaseStatementV1, - }, - /// The entry at `index` was applied and the durable cursor may advance to - /// `next_index`. This always follows the matching application event. - PhaseBatchCursorAdvanced { - outer: BlockReference, - index: usize, - next_index: usize, - }, - /// The local author fixed one exact carrier before authorizing its ECHO. - LocalCarrierFixed(BlockReference), - /// Bracha delivery became slot-global and immutable. - DeliveryLocked(BlockReference), - /// Existing non-durable output retained in its exact reducer order. - Effect(ModelEffect), -} - -/// Ordered typed input from which the executable model can be reconstructed. -/// -/// `CandidateRetained` is ordinary candidate-only retention. The stricter -/// `CandidateRecovered` variant additionally requires prior phase evidence, -/// matching [`RbcDagModel::recover_carrier`]. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ModelInputRecord { - CandidateRetained(CandidateCarrierV1), - CandidateRecovered(CandidateCarrierV1), - AuthenticatedIngress(AuthenticatedCarrierV1), - LocalCarrierFixed(LocallyAuthenticatedCarrierV1), - DataAvailable(BlockReference), -} - -#[derive(Default)] -struct TransitionLog { - trace: Vec, -} - -impl TransitionLog { - fn proof(&mut self, event: ModelTraceEvent) { - self.trace.push(event); - } - - fn effect(&mut self, effect: ModelEffect) { - self.trace.push(ModelTraceEvent::Effect(effect)); - } - - fn effects(&self) -> Vec { - self.trace - .iter() - .filter_map(|entry| match entry { - ModelTraceEvent::Effect(effect) => Some(effect.clone()), - _ => None, - }) - .collect() - } -} - -/// A transition evaluated against an immutable model revision. -/// -/// Fields are deliberately private: the only way to install the planned state -/// is [`RbcDagModel::commit_plan`], which rejects a stale or foreign base. -#[derive(Clone)] -pub struct ModelTransitionPlan { - base_revision: u64, - base_lineage: ModelLineage, - base_context: RbcDagContextV1, - base_authority: AuthorityIndex, - input: ModelInputRecord, - trace: Vec, - next_model: RbcDagModel, -} - -impl ModelTransitionPlan { - /// The typed reducer input must be durably recorded before the ordered - /// proof trace is persisted and this plan is committed. - pub fn input(&self) -> &ModelInputRecord { - &self.input - } - - pub fn trace(&self) -> &[ModelTraceEvent] { - &self.trace - } - - pub fn effects(&self) -> Vec { - self.trace - .iter() - .filter_map(|entry| match entry { - ModelTraceEvent::Effect(effect) => Some(effect.clone()), - _ => None, - }) - .collect() - } -} - /// Snapshot of the lifecycle predicates for one exact carrier. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct CarrierLifecycle { @@ -236,12 +122,6 @@ pub enum ModelError { previous: Option, proposed: Option, }, - RevisionOverflow, - StaleTransitionPlan { - expected_revision: u64, - actual_revision: u64, - }, - ForeignTransitionPlan, } impl fmt::Display for ModelError { @@ -330,8 +210,6 @@ pub struct RbcDagModel { committee_id: RbcDagCommitteeId, context: RbcDagContextV1, own_authority: AuthorityIndex, - revision: u64, - lineage: ModelLineage, local_carrier_round: RoundNumber, own_fixed: BTreeMap, carriers: BTreeMap, @@ -370,8 +248,6 @@ impl RbcDagModel { committee_id, context, own_authority, - revision: 0, - lineage: [0; 32], local_carrier_round: 1, own_fixed: BTreeMap::new(), carriers: BTreeMap::new(), @@ -394,127 +270,6 @@ impl RbcDagModel { self.context } - /// Monotonic reducer revision used to reject a plan computed from stale - /// state. It is advanced once per successfully applied typed input record. - pub fn revision(&self) -> u64 { - self.revision - } - - /// Evaluate one typed input against a clone without changing live state. - /// A durable adapter records [`ModelTransitionPlan::input`] first, then - /// [`ModelTransitionPlan::trace`] in order, and commits only after both - /// writes are durable. - pub fn plan_input(&self, input: ModelInputRecord) -> Result { - let mut next_model = self.clone(); - let log = next_model.apply_input_traced(input.clone())?; - Ok(ModelTransitionPlan { - base_revision: self.revision, - base_lineage: self.lineage, - base_context: self.context, - base_authority: self.own_authority, - input, - trace: log.trace, - next_model, - }) - } - - /// Atomically install a previously planned state after its ordered trace - /// has been durably recorded by the caller. - pub fn commit_plan( - &mut self, - plan: ModelTransitionPlan, - ) -> Result, ModelError> { - if self.context != plan.base_context || self.own_authority != plan.base_authority { - return Err(ModelError::ForeignTransitionPlan); - } - if self.revision != plan.base_revision { - return Err(ModelError::StaleTransitionPlan { - expected_revision: plan.base_revision, - actual_revision: self.revision, - }); - } - if self.lineage != plan.base_lineage { - return Err(ModelError::ForeignTransitionPlan); - } - let effects = plan.effects(); - *self = plan.next_model; - Ok(effects) - } - - /// Apply one typed record immediately. Runtime adapters that need - /// write-ahead durability should use [`Self::plan_input`] and - /// [`Self::commit_plan`] instead. - pub fn apply_input(&mut self, input: ModelInputRecord) -> Result, ModelError> { - self.apply_input_traced(input).map(|log| log.effects()) - } - - /// Deterministically reconstruct a model by replaying the original typed - /// inputs in their recorded order. No round or lock is synthesized. - pub fn replay_from_records( - committee: Arc, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - records: I, - ) -> Result<(Self, Vec), ModelError> - where - I: IntoIterator, - { - let mut model = Self::new(committee, own_authority, context)?; - let mut trace = Vec::new(); - for record in records { - trace.extend(model.apply_input_traced(record)?.trace); - } - Ok((model, trace)) - } - - fn apply_input_traced(&mut self, input: ModelInputRecord) -> Result { - let next_revision = self - .revision - .checked_add(1) - .ok_or(ModelError::RevisionOverflow)?; - let next_lineage = self.input_lineage(&input); - let mut log = TransitionLog::default(); - match input { - ModelInputRecord::CandidateRetained(carrier) => { - self.receive_carrier_traced( - carrier, - IngressAuthentication::CandidateOnly, - &mut log, - )?; - } - ModelInputRecord::CandidateRecovered(carrier) => { - self.recover_carrier_traced(carrier, &mut log)?; - } - ModelInputRecord::AuthenticatedIngress(authenticated) => { - self.receive_authenticated_traced(authenticated, &mut log)?; - } - ModelInputRecord::LocalCarrierFixed(authenticated) => { - self.start_local_carrier_traced(authenticated, &mut log)?; - } - ModelInputRecord::DataAvailable(reference) => { - self.mark_data_available_traced(reference, &mut log)?; - } - } - self.lineage = next_lineage; - self.revision = next_revision; - Ok(log) - } - - fn input_lineage(&self, input: &ModelInputRecord) -> ModelLineage { - let (kind, reference) = match input { - ModelInputRecord::CandidateRetained(carrier) => (0, carrier.reference()), - ModelInputRecord::CandidateRecovered(carrier) => (1, carrier.reference()), - ModelInputRecord::AuthenticatedIngress(carrier) => (2, carrier.reference()), - ModelInputRecord::LocalCarrierFixed(carrier) => (3, carrier.reference()), - ModelInputRecord::DataAvailable(reference) => (4, *reference), - }; - let mut hasher = Blake3Hasher::new_derive_key(MODEL_LINEAGE_DERIVE_CONTEXT); - hasher.update(&self.lineage); - hasher.update(&[kind]); - update_lineage_reference(&mut hasher, reference); - hasher.finalize().into() - } - /// Current sequential local carrier slot. If `can_create_carrier` is /// false, the local carrier is fixed and waits for exact-round quorum. pub fn local_carrier_round(&self) -> RoundNumber { @@ -598,14 +353,6 @@ impl RbcDagModel { &mut self, authenticated: LocallyAuthenticatedCarrierV1, ) -> Result, ModelError> { - self.apply_input(ModelInputRecord::LocalCarrierFixed(authenticated)) - } - - fn start_local_carrier_traced( - &mut self, - authenticated: LocallyAuthenticatedCarrierV1, - log: &mut TransitionLog, - ) -> Result<(), ModelError> { self.ensure_locally_authenticated(&authenticated)?; let carrier = authenticated.candidate().clone(); self.ensure_committee(&carrier)?; @@ -661,7 +408,6 @@ impl RbcDagModel { // ECHO is authorized or any embedded phase statement is exposed. self.preflight_receive(&carrier)?; self.own_fixed.insert(round, reference); - log.proof(ModelTraceEvent::LocalCarrierFixed(reference)); for statement in &expected_phase_batch { self.pending_phase_set.remove(statement); } @@ -673,9 +419,10 @@ impl RbcDagModel { (!selected_phase_indices.contains(&index)).then_some(statement) }) .collect(); - self.apply_received_carrier(carrier, IngressAuthentication::Authenticated, log); - self.maybe_advance_fast_clock(log); - Ok(()) + let mut effects = + self.apply_received_carrier(carrier, IngressAuthentication::Authenticated); + self.maybe_advance_fast_clock(&mut effects); + Ok(effects) } /// Stage canonical content without granting optimistic admission or ECHO. @@ -683,7 +430,7 @@ impl RbcDagModel { &mut self, carrier: CandidateCarrierV1, ) -> Result, ModelError> { - self.apply_input(ModelInputRecord::CandidateRetained(carrier)) + self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) } /// Admit a carrier only through the opaque capability produced by the @@ -692,36 +439,25 @@ impl RbcDagModel { &mut self, authenticated: AuthenticatedCarrierV1, ) -> Result, ModelError> { - self.apply_input(ModelInputRecord::AuthenticatedIngress(authenticated)) - } - - fn receive_authenticated_traced( - &mut self, - authenticated: AuthenticatedCarrierV1, - log: &mut TransitionLog, - ) -> Result<(), ModelError> { self.ensure_authenticated(&authenticated)?; if authenticated.candidate().header().author() == self.own_authority { return Err(ModelError::LocalCarrierRequiresStart( authenticated.candidate().reference(), )); } - self.receive_carrier_traced( + self.receive_carrier( authenticated.candidate().clone(), IngressAuthentication::Authenticated, - log, ) } - fn receive_carrier_traced( + fn receive_carrier( &mut self, carrier: CandidateCarrierV1, authentication: IngressAuthentication, - log: &mut TransitionLog, - ) -> Result<(), ModelError> { + ) -> Result, ModelError> { self.preflight_receive(&carrier)?; - self.apply_received_carrier(carrier, authentication, log); - Ok(()) + Ok(self.apply_received_carrier(carrier, authentication)) } fn preflight_receive(&self, carrier: &CandidateCarrierV1) -> Result<(), ModelError> { @@ -754,16 +490,16 @@ impl RbcDagModel { &mut self, carrier: CandidateCarrierV1, authentication: IngressAuthentication, - log: &mut TransitionLog, - ) { + ) -> Vec { let reference = carrier.reference(); self.carriers .entry(reference) .or_insert_with(|| CarrierRecord::new(carrier)); + let mut effects = Vec::new(); // Canonical content can satisfy a previously latched recovery even if // the receiver-specific authenticator is invalid. - self.drive_rbc(reference, log); + self.drive_rbc(reference, &mut effects); if authentication == IngressAuthentication::Authenticated { self.carriers @@ -779,11 +515,12 @@ impl RbcDagModel { } }; if selected && self.in_admission_window(reference.round) { - self.promote_authenticated(reference, log); + self.promote_authenticated(reference, &mut effects); } } - self.maybe_advance_fast_clock(log); - self.drain_delivered_phase_batches(log); + self.maybe_advance_fast_clock(&mut effects); + self.drain_delivered_phase_batches(&mut effects); + effects } /// Accept an exact recovered carrier only after authenticated phase @@ -792,14 +529,6 @@ impl RbcDagModel { &mut self, carrier: CandidateCarrierV1, ) -> Result, ModelError> { - self.apply_input(ModelInputRecord::CandidateRecovered(carrier)) - } - - fn recover_carrier_traced( - &mut self, - carrier: CandidateCarrierV1, - log: &mut TransitionLog, - ) -> Result<(), ModelError> { self.ensure_committee(&carrier)?; let reference = carrier.reference(); let key = (reference.round, reference.authority); @@ -810,7 +539,7 @@ impl RbcDagModel { if !expected { return Err(ModelError::UnexpectedRecovery(reference)); } - self.receive_carrier_traced(carrier, IngressAuthentication::CandidateOnly, log) + self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) } /// Record transaction-data availability established by the external @@ -821,20 +550,13 @@ impl RbcDagModel { &mut self, reference: BlockReference, ) -> Result, ModelError> { - self.apply_input(ModelInputRecord::DataAvailable(reference)) - } - - fn mark_data_available_traced( - &mut self, - reference: BlockReference, - log: &mut TransitionLog, - ) -> Result<(), ModelError> { self.carriers .get_mut(&reference) .ok_or(ModelError::MissingCarrier(reference))? .data_available = true; - self.drive_prefix(reference.authority, log); - Ok(()) + let mut effects = Vec::new(); + self.drive_prefix(reference.authority, &mut effects); + Ok(effects) } pub fn lifecycle(&self, reference: &BlockReference) -> Option { @@ -871,10 +593,6 @@ impl RbcDagModel { &mut self, frontier: &[Option], ) -> Result, ModelError> { - let next_revision = self - .revision - .checked_add(1) - .ok_or(ModelError::RevisionOverflow)?; if frontier.len() != self.committee.len() { return Err(ModelError::FrontierLength { expected: self.committee.len(), @@ -904,30 +622,9 @@ impl RbcDagModel { } self.included_frontier.clone_from_slice(frontier); self.included.extend(delta.iter().copied()); - self.advance_frontier_lineage(frontier); - self.revision = next_revision; Ok(delta.into_iter().collect()) } - fn advance_frontier_lineage(&mut self, frontier: &[Option]) { - let mut hasher = Blake3Hasher::new_derive_key(MODEL_LINEAGE_DERIVE_CONTEXT); - hasher.update(&self.lineage); - hasher.update(&[5]); - hasher.update(&(frontier.len() as u64).to_be_bytes()); - for reference in frontier { - match reference { - Some(reference) => { - hasher.update(&[1]); - update_lineage_reference(&mut hasher, *reference); - } - None => { - hasher.update(&[0]); - } - } - } - self.lineage = hasher.finalize().into(); - } - fn collect_frontier_extension( &self, authority: AuthorityIndex, @@ -1027,7 +724,7 @@ impl RbcDagModel { .or_default() } - fn authorize_local_echo(&mut self, reference: BlockReference, log: &mut TransitionLog) { + fn authorize_local_echo(&mut self, reference: BlockReference, effects: &mut Vec) { let own = self.own_authority; let slot = self.rbc_slot_mut(reference); if slot.echoed.is_some() { @@ -1040,10 +737,8 @@ impl RbcDagModel { .or_default() .echoes .insert(own); - let statement = RbcPhaseStatementV1::Echo { target: reference }; - log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); - self.queue_local_phase(statement); - self.drive_rbc(reference, log); + self.queue_local_phase(RbcPhaseStatementV1::Echo { target: reference }); + self.drive_rbc(reference, effects); } fn queue_local_phase(&mut self, statement: RbcPhaseStatementV1) { @@ -1052,14 +747,14 @@ impl RbcDagModel { } } - fn process_phase_batch(&mut self, outer: BlockReference, log: &mut TransitionLog) { - self.process_phase_batch_steps(outer, usize::MAX, log); - self.drain_delivered_phase_batches(log); + fn process_phase_batch(&mut self, outer: BlockReference, effects: &mut Vec) { + self.process_phase_batch_steps(outer, usize::MAX, effects); + self.drain_delivered_phase_batches(effects); } - fn drain_delivered_phase_batches(&mut self, log: &mut TransitionLog) { + fn drain_delivered_phase_batches(&mut self, effects: &mut Vec) { while let Some(outer) = self.pending_delivered_batch_replays.pop_front() { - self.process_phase_batch_steps(outer, usize::MAX, log); + self.process_phase_batch_steps(outer, usize::MAX, effects); } } @@ -1067,48 +762,32 @@ impl RbcDagModel { &mut self, outer: BlockReference, maximum_steps: usize, - log: &mut TransitionLog, + effects: &mut Vec, ) { let mut processed = 0; loop { if processed == maximum_steps { return; } - let Some((index, sender, statement)) = self.carriers.get(&outer).and_then(|record| { - let index = record.phase_batch_cursor; + let Some((sender, statement)) = self.carriers.get(&outer).and_then(|record| { record .carrier .header() .phase_batch() - .get(index) + .get(record.phase_batch_cursor) .copied() - .map(|statement| (index, record.carrier.header().author(), statement)) + .map(|statement| (record.carrier.header().author(), statement)) }) else { return; }; // Applying the statement is idempotent. Advance the persisted // cursor only afterwards, so a crash between the two replays the // same statement rather than skipping the unprocessed tail. - log.proof(ModelTraceEvent::PhaseBatchEntryApplied { - outer, - index, - sender, - statement, - }); - self.record_phase(sender, statement, log); - let next_index = { - let record = self - .carriers - .get_mut(&outer) - .expect("the outer carrier remains pinned"); - record.phase_batch_cursor += 1; - record.phase_batch_cursor - }; - log.proof(ModelTraceEvent::PhaseBatchCursorAdvanced { - outer, - index, - next_index, - }); + self.record_phase(sender, statement, effects); + self.carriers + .get_mut(&outer) + .expect("the outer carrier remains pinned") + .phase_batch_cursor += 1; processed += 1; } } @@ -1117,7 +796,7 @@ impl RbcDagModel { &mut self, sender: AuthorityIndex, statement: RbcPhaseStatementV1, - log: &mut TransitionLog, + effects: &mut Vec, ) { if !self.committee.known_authority(sender) { return; @@ -1159,10 +838,10 @@ impl RbcDagModel { candidate.readies.insert(sender); } } - self.drive_rbc(target, log); + self.drive_rbc(target, effects); } - fn drive_rbc(&mut self, target: BlockReference, log: &mut TransitionLog) { + fn drive_rbc(&mut self, target: BlockReference, effects: &mut Vec) { let slot_key = (target.round, target.authority); if !self .rbc_slots @@ -1229,7 +908,7 @@ impl RbcDagModel { .unwrap_or_default() .into_iter() .collect(); - log.effect(ModelEffect::NeedCarrier { target, holders }); + effects.push(ModelEffect::NeedCarrier { target, holders }); break; } RbcAction::SendReady => { @@ -1242,9 +921,7 @@ impl RbcDagModel { .or_default() .readies .insert(own); - let statement = RbcPhaseStatementV1::Ready { target }; - log.proof(ModelTraceEvent::LocalPhaseLocked(statement)); - self.queue_local_phase(statement); + self.queue_local_phase(RbcPhaseStatementV1::Ready { target }); } RbcAction::Deliver => { self.rbc_slot_mut(target).delivered = Some(target); @@ -1253,17 +930,16 @@ impl RbcDagModel { .get_mut(&target) .expect("delivery requires exact canonical carrier content"); record.delivered = true; - log.proof(ModelTraceEvent::DeliveryLocked(target)); - log.effect(ModelEffect::Delivered(target)); + effects.push(ModelEffect::Delivered(target)); self.pending_delivered_batch_replays.push_back(target); - self.drive_prefix(target.authority, log); + self.drive_prefix(target.authority, effects); } RbcAction::None => break, } } } - fn maybe_advance_fast_clock(&mut self, log: &mut TransitionLog) { + fn maybe_advance_fast_clock(&mut self, effects: &mut Vec) { let round = self.local_carrier_round; if !self.own_fixed.contains_key(&round) { return; @@ -1279,8 +955,8 @@ impl RbcDagModel { return; } self.local_carrier_round = round.saturating_add(1); - log.effect(ModelEffect::CarrierRoundAdvanced(self.local_carrier_round)); - self.promote_buffered_window(log); + effects.push(ModelEffect::CarrierRoundAdvanced(self.local_carrier_round)); + self.promote_buffered_window(effects); } fn in_admission_window(&self, round: RoundNumber) -> bool { @@ -1290,7 +966,7 @@ impl RbcDagModel { .saturating_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) } - fn promote_authenticated(&mut self, reference: BlockReference, log: &mut TransitionLog) { + fn promote_authenticated(&mut self, reference: BlockReference, effects: &mut Vec) { let slot_key = (reference.round, reference.authority); if self.authenticated_by_slot.get(&slot_key) != Some(&reference) || self @@ -1305,12 +981,11 @@ impl RbcDagModel { .get_mut(&reference) .expect("an authenticated carrier remains staged") .admitted = true; - log.proof(ModelTraceEvent::AdmissionLocked(reference)); - self.authorize_local_echo(reference, log); - self.process_phase_batch(reference, log); + self.authorize_local_echo(reference, effects); + self.process_phase_batch(reference, effects); } - fn promote_buffered_window(&mut self, log: &mut TransitionLog) { + fn promote_buffered_window(&mut self, effects: &mut Vec) { let eligible: Vec<_> = self .authenticated_by_slot .values() @@ -1318,11 +993,11 @@ impl RbcDagModel { .filter(|reference| self.in_admission_window(reference.round)) .collect(); for reference in eligible { - self.promote_authenticated(reference, log); + self.promote_authenticated(reference, effects); } } - fn drive_prefix(&mut self, authority: AuthorityIndex, log: &mut TransitionLog) { + fn drive_prefix(&mut self, authority: AuthorityIndex, effects: &mut Vec) { loop { let Some(current_tip) = self.prefix_tips.get(authority as usize).copied() else { return; @@ -1346,7 +1021,7 @@ impl RbcDagModel { .expect("delivered carrier exists") .prefix_closed = true; self.prefix_tips[authority as usize] = next; - log.effect(ModelEffect::PrefixAdvanced { + effects.push(ModelEffect::PrefixAdvanced { authority, tip: next, }); @@ -1354,12 +1029,6 @@ impl RbcDagModel { } } -fn update_lineage_reference(hasher: &mut Blake3Hasher, reference: BlockReference) { - hasher.update(&reference.authority.to_be_bytes()); - hasher.update(&reference.round.to_be_bytes()); - hasher.update(reference.digest.as_array()); -} - #[cfg(test)] mod tests { use super::*; @@ -1563,345 +1232,6 @@ mod tests { all_honest_progress(7); } - #[test] - fn planned_authenticated_ingress_locks_admission_before_echo() { - let committee = committee(4); - let model = model(Arc::clone(&committee), 3); - let (own_prev, weak_parents) = genesis_parents(&committee, 0); - let carrier = - candidate(&committee, 0, 1, own_prev, weak_parents, Vec::new(), 0xD0).unwrap(); - let target = carrier.reference(); - let authenticated = authenticate_for(&committee, &carrier, 3); - let plan = model - .plan_input(ModelInputRecord::AuthenticatedIngress(authenticated)) - .unwrap(); - - let admission = plan - .trace() - .iter() - .position(|event| *event == ModelTraceEvent::AdmissionLocked(target)) - .unwrap(); - let echo = plan - .trace() - .iter() - .position(|event| { - *event == ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Echo { target }) - }) - .unwrap(); - assert!(admission < echo); - assert_eq!(model.revision(), 0); - assert!(model.lifecycle(&target).is_none()); - } - - #[test] - fn phase_application_precedes_ready_and_ready_precedes_delivery() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (own_prev, weak_parents) = genesis_parents(&committee, 0); - let target_carrier = - candidate(&committee, 0, 1, own_prev, weak_parents, Vec::new(), 0xD1).unwrap(); - let target = target_carrier.reference(); - model.stage_candidate(target_carrier).unwrap(); - // One remote READY is below V. The enclosing carrier supplies the - // second; the resulting local READY is then the third vote and delivers. - let mut setup = TransitionLog::default(); - model.record_phase(0, RbcPhaseStatementV1::Ready { target }, &mut setup); - - let outer = candidate( - &committee, - 1, - 2, - BlockReference::new_test(1, 1), - vec![ - BlockReference::new_test(0, 1), - BlockReference::new_test(2, 1), - ], - vec![RbcPhaseStatementV1::Ready { target }], - 0xD2, - ) - .unwrap(); - let outer_reference = outer.reference(); - let authenticated = authenticate_for(&committee, &outer, 3); - let plan = model - .plan_input(ModelInputRecord::AuthenticatedIngress(authenticated)) - .unwrap(); - let trace = plan.trace(); - - let applied = trace - .iter() - .position(|event| { - matches!( - event, - ModelTraceEvent::PhaseBatchEntryApplied { - outer, - index: 0, - sender: 1, - statement: RbcPhaseStatementV1::Ready { target: actual }, - } if *outer == outer_reference && *actual == target - ) - }) - .unwrap(); - let ready = trace - .iter() - .position(|event| { - *event == ModelTraceEvent::LocalPhaseLocked(RbcPhaseStatementV1::Ready { target }) - }) - .unwrap(); - let delivery = trace - .iter() - .position(|event| *event == ModelTraceEvent::DeliveryLocked(target)) - .unwrap(); - let cursor = trace - .iter() - .position(|event| { - matches!( - event, - ModelTraceEvent::PhaseBatchCursorAdvanced { - outer, - index: 0, - next_index: 1, - } if *outer == outer_reference - ) - }) - .unwrap(); - assert!(applied < ready); - assert!(ready < delivery); - assert!(delivery < cursor); - } - - #[test] - fn delivery_lock_precedes_replay_of_the_delivered_carrier_batch() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let replay_target = BlockReference::new_test(2, 1); - let target_carrier = candidate( - &committee, - 0, - 2, - BlockReference::new_test(0, 1), - vec![ - BlockReference::new_test(1, 1), - BlockReference::new_test(2, 1), - ], - vec![RbcPhaseStatementV1::Echo { - target: replay_target, - }], - 0xD3, - ) - .unwrap(); - let target = target_carrier.reference(); - model.stage_candidate(target_carrier).unwrap(); - let mut setup = TransitionLog::default(); - model.record_phase(0, RbcPhaseStatementV1::Ready { target }, &mut setup); - - let outer = candidate( - &committee, - 1, - 3, - BlockReference::new_test(1, 2), - vec![ - BlockReference::new_test(0, 2), - BlockReference::new_test(2, 2), - ], - vec![RbcPhaseStatementV1::Ready { target }], - 0xD4, - ) - .unwrap(); - let authenticated = authenticate_for(&committee, &outer, 3); - let plan = model - .plan_input(ModelInputRecord::AuthenticatedIngress(authenticated)) - .unwrap(); - let trace = plan.trace(); - let delivery = trace - .iter() - .position(|event| *event == ModelTraceEvent::DeliveryLocked(target)) - .unwrap(); - let replay = trace - .iter() - .position(|event| { - matches!( - event, - ModelTraceEvent::PhaseBatchEntryApplied { - outer, - index: 0, - sender: 0, - statement: RbcPhaseStatementV1::Echo { target: actual }, - } if *outer == target && *actual == replay_target - ) - }) - .unwrap(); - assert!(delivery < replay); - } - - #[test] - fn planned_transition_is_rollback_safe_and_rejects_stale_or_divergent_commits() { - let committee = committee(4); - let mut live = model(Arc::clone(&committee), 3); - let (own_prev, weak_parents) = genesis_parents(&committee, 0); - let carrier = - candidate(&committee, 0, 1, own_prev, weak_parents, Vec::new(), 0xD5).unwrap(); - let reference = carrier.reference(); - let authenticated = authenticate_for(&committee, &carrier, 3); - let plan = live - .plan_input(ModelInputRecord::AuthenticatedIngress( - authenticated.clone(), - )) - .unwrap(); - - // Planning and dropping a clone cannot expose any live transition. - assert_eq!(live.revision(), 0); - assert!(live.lifecycle(&reference).is_none()); - let mut committed = live.clone(); - committed.commit_plan(plan.clone()).unwrap(); - assert_eq!(committed.revision(), 1); - assert!(committed.lifecycle(&reference).unwrap().admitted); - assert!(live.lifecycle(&reference).is_none()); - - // Any intervening successful input invalidates the old base revision. - live.stage_candidate(carrier).unwrap(); - assert_eq!(live.revision(), 1); - assert_eq!( - live.commit_plan(plan), - Err(ModelError::StaleTransitionPlan { - expected_revision: 0, - actual_revision: 1, - }) - ); - let lifecycle = live.lifecycle(&reference).unwrap(); - assert!(!lifecycle.authenticated); - assert!(!lifecycle.admitted); - - // Equal revision numbers do not make independently evolved clones - // interchangeable: their private lineages bind a plan to its exact - // base state. - let divergent_plan = committed - .plan_input(ModelInputRecord::DataAvailable(reference)) - .unwrap(); - assert_eq!( - live.commit_plan(divergent_plan), - Err(ModelError::ForeignTransitionPlan) - ); - let lifecycle = live.lifecycle(&reference).unwrap(); - assert!(!lifecycle.authenticated); - assert!(!lifecycle.data_available); - } - - #[test] - fn ordered_typed_replay_reconstructs_rounds_and_local_locks() { - let committee = committee(4); - let context = context(&committee); - let mut live = RbcDagModel::new(Arc::clone(&committee), 3, context).unwrap(); - let mut records = Vec::new(); - let mut references = Vec::new(); - - let (own_prev, weak_parents) = live.local_parent_set().unwrap(); - let local_one = candidate( - &committee, - 3, - 1, - own_prev, - weak_parents, - live.pending_phase_batch(), - 0xE0, - ) - .unwrap(); - let local_one_record = - ModelInputRecord::LocalCarrierFixed(authenticate_local(&committee, &local_one)); - live.apply_input(local_one_record.clone()).unwrap(); - records.push(local_one_record); - references.push(local_one.reference()); - - let mut round_one = BTreeMap::new(); - for (author, marker) in [(0, 0xE1), (1, 0xE2)] { - let (own_prev, weak_parents) = genesis_parents(&committee, author); - let carrier = candidate( - &committee, - author, - 1, - own_prev, - weak_parents, - Vec::new(), - marker, - ) - .unwrap(); - if author == 0 { - let retained = ModelInputRecord::CandidateRetained(carrier.clone()); - live.apply_input(retained.clone()).unwrap(); - records.push(retained); - } - let ingress = - ModelInputRecord::AuthenticatedIngress(authenticate_for(&committee, &carrier, 3)); - live.apply_input(ingress.clone()).unwrap(); - records.push(ingress); - references.push(carrier.reference()); - round_one.insert(author, carrier); - } - assert_eq!(live.local_carrier_round(), 2); - - let (own_prev, weak_parents) = live.local_parent_set().unwrap(); - let local_two = candidate( - &committee, - 3, - 2, - own_prev, - weak_parents, - live.pending_phase_batch(), - 0xE3, - ) - .unwrap(); - let local_two_record = - ModelInputRecord::LocalCarrierFixed(authenticate_local(&committee, &local_two)); - live.apply_input(local_two_record.clone()).unwrap(); - records.push(local_two_record); - references.push(local_two.reference()); - - for (author, marker, other) in [(0, 0xE4, 1), (1, 0xE5, 0)] { - let carrier = candidate( - &committee, - author, - 2, - round_one[&author].reference(), - vec![round_one[&other].reference(), local_one.reference()], - Vec::new(), - marker, - ) - .unwrap(); - let ingress = - ModelInputRecord::AuthenticatedIngress(authenticate_for(&committee, &carrier, 3)); - live.apply_input(ingress.clone()).unwrap(); - records.push(ingress); - references.push(carrier.reference()); - } - assert_eq!(live.local_carrier_round(), 3); - - let (replayed, trace) = - RbcDagModel::replay_from_records(Arc::clone(&committee), 3, context, records.clone()) - .unwrap(); - assert!(!trace.is_empty()); - assert_eq!(replayed.revision(), records.len() as u64); - assert_eq!(replayed.lineage, live.lineage); - assert_eq!(replayed.local_carrier_round, live.local_carrier_round); - assert_eq!(replayed.own_fixed, live.own_fixed); - assert_eq!(replayed.authenticated_by_slot, live.authenticated_by_slot); - assert_eq!(replayed.admitted_by_slot, live.admitted_by_slot); - assert_eq!(replayed.rbc_slots, live.rbc_slots); - assert_eq!(replayed.pending_phases, live.pending_phases); - assert_eq!(replayed.pending_phase_set, live.pending_phase_set); - for reference in references { - assert_eq!(replayed.lifecycle(&reference), live.lifecycle(&reference)); - } - - // Recovery is sequential: retaining only the round-two local record - // cannot synthesize round one or jump the local carrier clock. - assert!(matches!( - RbcDagModel::replay_from_records(committee, 3, context, [records[4].clone()],), - Err(ModelError::UnexpectedLocalRound { - expected: 1, - actual: 2, - }) - )); - } - #[test] fn phase_backlog_exposes_only_a_bounded_fifo_prefix() { let committee = committee(4); @@ -1972,10 +1302,10 @@ mod tests { sender: AuthorityIndex, statement: RbcPhaseStatementV1, ) -> Vec { - let mut log = TransitionLog::default(); - model.record_phase(sender, statement, &mut log); - model.drain_delivered_phase_batches(&mut log); - log.effects() + let mut effects = Vec::new(); + model.record_phase(sender, statement, &mut effects); + model.drain_delivered_phase_batches(&mut effects); + effects } fn force_deliver(model: &mut RbcDagModel, carrier: CandidateCarrierV1) { @@ -2455,7 +1785,7 @@ mod tests { model.stage_candidate(outer).unwrap(); let mut uninterrupted = model.clone(); - uninterrupted.process_phase_batch(outer_ref, &mut TransitionLog::default()); + uninterrupted.process_phase_batch(outer_ref, &mut Vec::new()); // Model a crash after the first idempotent statement was persisted but // before the outer batch cursor was advanced. @@ -2463,9 +1793,9 @@ mod tests { restarted.record_phase( 0, RbcPhaseStatementV1::Echo { target: first }, - &mut TransitionLog::default(), + &mut Vec::new(), ); - restarted.process_phase_batch(outer_ref, &mut TransitionLog::default()); + restarted.process_phase_batch(outer_ref, &mut Vec::new()); assert_eq!(restarted.rbc_slots, uninterrupted.rbc_slots); assert_eq!(restarted.pending_phases, uninterrupted.pending_phases); @@ -2525,7 +1855,8 @@ mod tests { model .pending_delivered_batch_replays .push_back(*references.last().unwrap()); - model.drain_delivered_phase_batches(&mut TransitionLog::default()); + let mut effects = Vec::new(); + model.drain_delivered_phase_batches(&mut effects); assert!(model.pending_delivered_batch_replays.is_empty()); assert_eq!(model.delivered(0, 1), Some(references[0])); diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs index 1bda66fe..86fc5274 100644 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ b/crates/starfish-core/src/starfish_rbc_dag/projection.rs @@ -22,7 +22,7 @@ use crate::{ use super::{ CandidateCarrierV1, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, - RbcDagCommitteeContextV1, RbcDagProjectionError, carrier_genesis_reference, + RbcDagCommitteeId, RbcDagProjectionError, carrier_genesis_reference, }; /// An indexed exact carrier-prefix frontier. `None` is the authority's virtual @@ -135,7 +135,7 @@ struct ProjectedVertex { #[derive(Clone)] pub struct CertifiedProjectionModel { committee: Arc, - committee_context: RbcDagCommitteeContextV1, + committee_id: RbcDagCommitteeId, carriers: BTreeMap, delivered_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, closed_prefixes: Vec>, @@ -146,21 +146,13 @@ pub struct CertifiedProjectionModel { } impl CertifiedProjectionModel { - /// Convenience constructor that validates and hashes the committee once. - /// Runtime code that already owns the reusable capability should call - /// [`Self::from_committee_context`]. pub fn new(committee: Arc) -> Result { - let committee = RbcDagCommitteeContextV1::new(committee) + let committee_id = RbcDagCommitteeId::derive(&committee) .map_err(|_| CertifiedProjectionError::CommitteeMismatch)?; - Ok(Self::from_committee_context(committee)) - } - - pub fn from_committee_context(committee: RbcDagCommitteeContextV1) -> Self { - let committee_size = committee.committee().len(); - let committee_arc = committee.committee_arc(); - Self { - committee: committee_arc, - committee_context: committee, + let committee_size = committee.len(); + Ok(Self { + committee, + committee_id, carriers: BTreeMap::new(), delivered_slots: BTreeMap::new(), closed_prefixes: vec![Vec::new(); committee_size], @@ -168,7 +160,7 @@ impl CertifiedProjectionModel { consensus_slots: BTreeMap::new(), committed_frontier: vec![None; committee_size], committed_anchors: BTreeSet::new(), - } + }) } /// Retain a canonical carrier independently of optional-vertex validity. @@ -176,7 +168,7 @@ impl CertifiedProjectionModel { &mut self, candidate: CandidateCarrierV1, ) -> Result<(), CertifiedProjectionError> { - if candidate.committee_id() != self.committee_context.committee_id() { + if candidate.committee_id() != self.committee_id { return Err(CertifiedProjectionError::CommitteeMismatch); } self.carriers @@ -301,7 +293,7 @@ impl CertifiedProjectionModel { state .candidate - .validate_consensus_vertex_with_committee(&self.committee_context) + .validate_consensus_vertex(&self.committee) .map_err(CertifiedProjectionError::InvalidProjectionShape)?; if !state.delivered { return Err(CertifiedProjectionError::CarrierNotDelivered( diff --git a/crates/starfish-core/src/starfish_rbc_dag/storage.rs b/crates/starfish-core/src/starfish_rbc_dag/storage.rs deleted file mode 100644 index d553658a..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag/storage.rs +++ /dev/null @@ -1,1388 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Durable, opaque-record WAL for Starfish-RBC-DAG shadow state. -//! -//! The storage layer deliberately does not encode [`super::journal::JournalEventV1`]. -//! A later integration layer owns that versioned codec and the recovery of -//! opaque authentication capabilities. This module supplies the durability -//! boundary underneath it: one batch is one checksummed frame, and a caller -//! may expose the corresponding effects only after [`ShadowWalV1::append_batch`] -//! returns successfully. -//! -//! Recovery discards only a physically short final frame. A fully present -//! frame with a bad header, commit marker, or checksum is reported as -//! corruption even at end-of-file, so acknowledged proof-critical state is -//! never silently erased. - -use std::{ - error::Error, - ffi::OsString, - fmt, - fs::{self, File, OpenOptions}, - io::{self, Read, Seek, SeekFrom, Write}, - path::{Path, PathBuf}, - sync::atomic::{AtomicU64, Ordering}, -}; - -#[cfg(unix)] -use std::{ffi::CString, os::unix::ffi::OsStrExt}; - -use crate::types::AuthorityIndex; - -use super::RbcDagContextV1; - -const FILE_MAGIC: &[u8; 16] = b"STRFSH_RBCDAGWAL"; -const FRAME_MAGIC: u32 = 0x5242_4446; // "RBDF" -const FRAME_COMMIT_MAGIC: u32 = 0x434F_4D54; // "COMT" - -pub const SHADOW_WAL_FORMAT_VERSION_V1: u16 = 1; -pub const MAX_SHADOW_WAL_RECORD_SIZE_V1: usize = 16 * 1024 * 1024; -pub const MAX_SHADOW_WAL_BATCH_RECORDS_V1: usize = 4_096; -pub const MAX_SHADOW_WAL_FRAME_PAYLOAD_V1: usize = 64 * 1024 * 1024; - -const FILE_HEADER_PREFIX_LEN: usize = 16 + 2 + 2 + 32 + 32 + 2 + 2; -const FILE_HEADER_LEN: usize = FILE_HEADER_PREFIX_LEN + 4; -const FRAME_HEADER_PREFIX_LEN: usize = 4 + 2 + 2 + 8 + 8 + 4 + 4; -const FRAME_HEADER_LEN: usize = FRAME_HEADER_PREFIX_LEN + 4; -const FRAME_TRAILER_LEN: usize = 4 + 4; -const MAX_INITIALIZATION_TEMP_ATTEMPTS: usize = 128; - -static INITIALIZATION_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct ShadowWalNamespaceV1 { - protocol_instance: [u8; 32], - committee_id: [u8; 32], - own_authority: AuthorityIndex, -} - -impl ShadowWalNamespaceV1 { - pub fn new(context: RbcDagContextV1, own_authority: AuthorityIndex) -> Self { - Self { - protocol_instance: *context.protocol_instance().as_bytes(), - committee_id: *context.committee_id().as_bytes(), - own_authority, - } - } - - pub fn protocol_instance(&self) -> &[u8; 32] { - &self.protocol_instance - } - - pub fn committee_id(&self) -> &[u8; 32] { - &self.committee_id - } - - pub fn own_authority(&self) -> AuthorityIndex { - self.own_authority - } - - pub fn format_version(&self) -> u16 { - SHADOW_WAL_FORMAT_VERSION_V1 - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct RecoveredBatchV1 { - sequence: u64, - start_offset: u64, - end_offset: u64, - records: Vec>, -} - -impl RecoveredBatchV1 { - pub fn sequence(&self) -> u64 { - self.sequence - } - - pub fn start_offset(&self) -> u64 { - self.start_offset - } - - pub fn end_offset(&self) -> u64 { - self.end_offset - } - - pub fn records(&self) -> &[Vec] { - &self.records - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ShadowWalRecoveryV1 { - batches: Vec, - durable_file_len: u64, - record_count: u64, - discarded_tail_bytes: u64, -} - -impl ShadowWalRecoveryV1 { - pub fn batches(&self) -> &[RecoveredBatchV1] { - &self.batches - } - - pub fn batch_count(&self) -> u64 { - u64::try_from(self.batches.len()).expect("batch count always fits u64") - } - - pub fn record_count(&self) -> u64 { - self.record_count - } - - pub fn durable_file_len(&self) -> u64 { - self.durable_file_len - } - - pub fn discarded_tail_bytes(&self) -> u64 { - self.discarded_tail_bytes - } - - pub fn records(&self) -> Vec<&[u8]> { - self.batches - .iter() - .flat_map(|batch| batch.records.iter().map(Vec::as_slice)) - .collect() - } - - pub fn into_records(self) -> Vec> { - self.batches - .into_iter() - .flat_map(|batch| batch.records) - .collect() - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct DurableBatchPositionV1 { - sequence: u64, - start_offset: u64, - end_offset: u64, - record_count: u32, -} - -impl DurableBatchPositionV1 { - pub fn sequence(&self) -> u64 { - self.sequence - } - - pub fn start_offset(&self) -> u64 { - self.start_offset - } - - pub fn end_offset(&self) -> u64 { - self.end_offset - } - - pub fn record_count(&self) -> u32 { - self.record_count - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct ShadowWalSummaryV1 { - file_len: u64, - batch_count: u64, - record_count: u64, -} - -impl ShadowWalSummaryV1 { - pub fn file_len(&self) -> u64 { - self.file_len - } - - pub fn batch_count(&self) -> u64 { - self.batch_count - } - - pub fn record_count(&self) -> u64 { - self.record_count - } -} - -#[derive(Debug)] -pub enum ShadowWalErrorV1 { - Io(io::Error), - TruncatedFileHeader { - actual: u64, - expected: usize, - }, - InvalidFileMagic, - UnsupportedFileVersion(u16), - InvalidFileHeaderLength(u16), - InvalidFileHeaderFlags(u16), - InvalidFileHeaderChecksum, - NamespaceMismatch, - EmptyBatch, - TooManyRecords(usize), - RecordTooLarge(usize), - FramePayloadTooLarge(usize), - LengthOverflow, - CorruptFrame { - offset: u64, - reason: &'static str, - }, - UnexpectedFrameSequence { - offset: u64, - expected: u64, - actual: u64, - }, - ExternalFileMutation { - expected: u64, - actual: u64, - }, - Poisoned, -} - -impl fmt::Display for ShadowWalErrorV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(error) => write!(formatter, "Starfish-RBC-DAG shadow WAL I/O error: {error}"), - other => write!(formatter, "Starfish-RBC-DAG shadow WAL error: {other:?}"), - } - } -} - -impl Error for ShadowWalErrorV1 { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(error) => Some(error), - _ => None, - } - } -} - -impl From for ShadowWalErrorV1 { - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -pub struct ShadowWalV1 { - path: PathBuf, - file: File, - namespace: ShadowWalNamespaceV1, - durable_file_len: u64, - batch_count: u64, - record_count: u64, - poisoned: bool, -} - -impl ShadowWalV1 { - /// Open the WAL for exclusive single-writer use and replay every complete - /// durable batch in sequence order. - /// - /// The caller must ensure that no other writer mutates `path` while this - /// handle is alive. A changed file length is detected before an append, - /// but this adapter intentionally does not provide cross-process locking. - pub fn open( - path: impl AsRef, - namespace: ShadowWalNamespaceV1, - ) -> Result<(Self, ShadowWalRecoveryV1), ShadowWalErrorV1> { - let path = path.as_ref().to_path_buf(); - if let Some(parent) = nonempty_parent(&path) { - create_parent_directories_durable(parent)?; - } - - let mut file = loop { - match OpenOptions::new().read(true).write(true).open(&path) { - Ok(file) => break file, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - match fs::symlink_metadata(&path) { - Ok(_) => { - // A directory entry (for example a dangling - // symlink) already exists. Treat it as malformed; - // otherwise a no-replace publication retry would - // spin forever and might later mask the entry. - return Err(error.into()); - } - Err(metadata_error) if metadata_error.kind() == io::ErrorKind::NotFound => { - } - Err(metadata_error) => return Err(metadata_error.into()), - } - if let Some(file) = initialize_new_wal(&path, namespace)? { - break file; - } - // Another initializer atomically published `path` first. - // Open and validate exactly what won the race; never - // replace or repair it here. - } - Err(error) => return Err(error.into()), - } - }; - - let recovery = recover_file(&mut file, namespace)?; - file.seek(SeekFrom::Start(recovery.durable_file_len))?; - let wal = Self { - path, - file, - namespace, - durable_file_len: recovery.durable_file_len, - batch_count: recovery.batch_count(), - record_count: recovery.record_count, - poisoned: false, - }; - Ok((wal, recovery)) - } - - pub fn path(&self) -> &Path { - &self.path - } - - pub fn namespace(&self) -> ShadowWalNamespaceV1 { - self.namespace - } - - pub fn file_len(&self) -> u64 { - self.durable_file_len - } - - pub fn batch_count(&self) -> u64 { - self.batch_count - } - - pub fn record_count(&self) -> u64 { - self.record_count - } - - pub fn is_poisoned(&self) -> bool { - self.poisoned - } - - /// Append and fsync one atomic record batch before returning. - /// - /// Any seek, write, or fsync failure poisons this handle because the commit - /// result may be ambiguous. Drop it and reopen the WAL; recovery will - /// either retain the complete frame or discard its physically short torn - /// suffix. - pub fn append_batch( - &mut self, - records: &[Vec], - ) -> Result { - if self.poisoned { - return Err(ShadowWalErrorV1::Poisoned); - } - let sequence = self.batch_count; - let frame = encode_frame(sequence, records)?; - let added_records = - u64::try_from(records.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; - let next_batch_count = self - .batch_count - .checked_add(1) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - let next_record_count = self - .record_count - .checked_add(added_records) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - let durable_record_count = - u32::try_from(records.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; - let actual_len = self.file.metadata()?.len(); - if actual_len != self.durable_file_len { - self.poisoned = true; - return Err(ShadowWalErrorV1::ExternalFileMutation { - expected: self.durable_file_len, - actual: actual_len, - }); - } - let start_offset = self.durable_file_len; - let frame_len = u64::try_from(frame.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; - let end_offset = start_offset - .checked_add(frame_len) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - - if let Err(error) = self - .file - .seek(SeekFrom::Start(start_offset)) - .and_then(|_| self.file.write_all(&frame)) - .and_then(|_| self.file.sync_all()) - { - self.poisoned = true; - return Err(ShadowWalErrorV1::Io(error)); - } - - self.durable_file_len = end_offset; - self.batch_count = next_batch_count; - self.record_count = next_record_count; - Ok(DurableBatchPositionV1 { - sequence, - start_offset, - end_offset, - record_count: durable_record_count, - }) - } - - pub fn summary(&self) -> ShadowWalSummaryV1 { - ShadowWalSummaryV1 { - file_len: self.durable_file_len, - batch_count: self.batch_count, - record_count: self.record_count, - } - } - - /// Flush file metadata and close this writer by consuming it. - pub fn shutdown(self) -> Result { - self.file.sync_all()?; - if self.poisoned { - return Err(ShadowWalErrorV1::Poisoned); - } - Ok(self.summary()) - } -} - -/// Build a complete, durable header away from the canonical path, then -/// publish it without replacing any concurrently-created target. -/// -/// `Ok(None)` means another initializer won the publication race. The caller -/// must open and validate that target rather than assuming it is compatible. -fn initialize_new_wal( - path: &Path, - namespace: ShadowWalNamespaceV1, -) -> Result, ShadowWalErrorV1> { - let (temporary_path, mut file) = create_initialization_temp(path)?; - let cleanup = InitializationTempCleanup::new(temporary_path.clone()); - file.write_all(&encode_file_header(namespace))?; - file.sync_all()?; - - match atomic_rename_noreplace(&temporary_path, path) { - Ok(()) => { - // The rename makes the complete inode visible atomically; the - // directory sync makes that name durable across a power loss. - sync_parent_directory(path)?; - cleanup.disarm(); - Ok(Some(file)) - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => Ok(None), - Err(error) => Err(error.into()), - } -} - -fn create_initialization_temp(path: &Path) -> io::Result<(PathBuf, File)> { - let parent = nonempty_parent(path).unwrap_or_else(|| Path::new(".")); - let file_name = path.file_name().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "shadow WAL path has no file name", - ) - })?; - for _ in 0..MAX_INITIALIZATION_TEMP_ATTEMPTS { - let counter = INITIALIZATION_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); - let mut temporary_name = OsString::from("."); - temporary_name.push(file_name); - temporary_name.push(format!(".init-{}-{counter}.tmp", std::process::id())); - let temporary_path = parent.join(temporary_name); - match OpenOptions::new() - .create_new(true) - .read(true) - .write(true) - .open(&temporary_path) - { - Ok(file) => return Ok((temporary_path, file)), - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error), - } - } - Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "could not allocate a unique shadow WAL initialization file", - )) -} - -struct InitializationTempCleanup { - path: PathBuf, - armed: bool, -} - -impl InitializationTempCleanup { - fn new(path: PathBuf) -> Self { - Self { path, armed: true } - } - - fn disarm(mut self) { - self.armed = false; - } -} - -impl Drop for InitializationTempCleanup { - fn drop(&mut self) { - if self.armed { - let _ = fs::remove_file(&self.path); - } - } -} - -#[cfg(any(target_os = "linux", target_os = "android"))] -fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { - let from = path_to_c_string(from)?; - let to = path_to_c_string(to)?; - // SAFETY: both paths are live NUL-terminated C strings for the duration - // of the call. RENAME_NOREPLACE gives the required atomic no-clobber - // publication semantics. - let result = unsafe { - libc::renameat2( - libc::AT_FDCWD, - from.as_ptr(), - libc::AT_FDCWD, - to.as_ptr(), - libc::RENAME_NOREPLACE, - ) - }; - if result == 0 { - Ok(()) - } else { - let error = io::Error::last_os_error(); - if matches!(error.raw_os_error(), Some(code) - if code == libc::ENOSYS - || code == libc::EINVAL - || code == libc::ENOTSUP - || code == libc::EOPNOTSUPP) - { - atomic_link_noreplace(from.as_bytes(), to.as_bytes()) - } else { - Err(error) - } - } -} - -#[cfg(target_vendor = "apple")] -fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { - let from = path_to_c_string(from)?; - let to = path_to_c_string(to)?; - // SAFETY: both paths are live NUL-terminated C strings for the duration - // of the call. RENAME_EXCL prevents replacement of an existing target. - let result = unsafe { libc::renamex_np(from.as_ptr(), to.as_ptr(), libc::RENAME_EXCL) }; - if result == 0 { - Ok(()) - } else { - let error = io::Error::last_os_error(); - if matches!(error.raw_os_error(), Some(code) if code == libc::ENOTSUP || code == libc::EINVAL) - { - atomic_link_noreplace(from.as_bytes(), to.as_bytes()) - } else { - Err(error) - } - } -} - -#[cfg(unix)] -fn path_to_c_string(path: &Path) -> io::Result { - CString::new(path.as_os_str().as_bytes()).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - "shadow WAL path contains an interior NUL", - ) - }) -} - -#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))] -fn atomic_link_noreplace(from: &[u8], to: &[u8]) -> io::Result<()> { - let from = Path::new(std::ffi::OsStr::from_bytes(from)); - let to = Path::new(std::ffi::OsStr::from_bytes(to)); - fs::hard_link(from, to)?; - // The target now atomically names the fully-synced inode. Failure to - // remove the private temporary name is harmless to WAL correctness. - let _ = fs::remove_file(from); - Ok(()) -} - -#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))] -fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { - // Portable fallback for platforms without a no-replace rename primitive: - // same-directory hard-link creation is still an atomic no-clobber publish. - fs::hard_link(from, to)?; - let _ = fs::remove_file(from); - Ok(()) -} - -fn encode_file_header(namespace: ShadowWalNamespaceV1) -> [u8; FILE_HEADER_LEN] { - let mut header = [0u8; FILE_HEADER_LEN]; - header[0..16].copy_from_slice(FILE_MAGIC); - header[16..18].copy_from_slice(&SHADOW_WAL_FORMAT_VERSION_V1.to_be_bytes()); - header[18..20].copy_from_slice(&(FILE_HEADER_LEN as u16).to_be_bytes()); - header[20..52].copy_from_slice(&namespace.protocol_instance); - header[52..84].copy_from_slice(&namespace.committee_id); - header[84..86].copy_from_slice(&namespace.own_authority.to_be_bytes()); - header[86..88].copy_from_slice(&0u16.to_be_bytes()); - let checksum = crc32c(&header[..FILE_HEADER_PREFIX_LEN]); - header[FILE_HEADER_PREFIX_LEN..FILE_HEADER_LEN].copy_from_slice(&checksum.to_be_bytes()); - header -} - -fn validate_file_header( - header: &[u8; FILE_HEADER_LEN], - expected_namespace: ShadowWalNamespaceV1, -) -> Result<(), ShadowWalErrorV1> { - if &header[0..16] != FILE_MAGIC { - return Err(ShadowWalErrorV1::InvalidFileMagic); - } - let version = read_u16(&header[16..18]); - if version != SHADOW_WAL_FORMAT_VERSION_V1 { - return Err(ShadowWalErrorV1::UnsupportedFileVersion(version)); - } - let header_len = read_u16(&header[18..20]); - if usize::from(header_len) != FILE_HEADER_LEN { - return Err(ShadowWalErrorV1::InvalidFileHeaderLength(header_len)); - } - let flags = read_u16(&header[86..88]); - if flags != 0 { - return Err(ShadowWalErrorV1::InvalidFileHeaderFlags(flags)); - } - let checksum = read_u32(&header[FILE_HEADER_PREFIX_LEN..FILE_HEADER_LEN]); - if checksum != crc32c(&header[..FILE_HEADER_PREFIX_LEN]) { - return Err(ShadowWalErrorV1::InvalidFileHeaderChecksum); - } - let actual_namespace = ShadowWalNamespaceV1 { - protocol_instance: header[20..52] - .try_into() - .expect("fixed protocol-instance range"), - committee_id: header[52..84].try_into().expect("fixed committee range"), - own_authority: read_u16(&header[84..86]), - }; - if actual_namespace != expected_namespace { - return Err(ShadowWalErrorV1::NamespaceMismatch); - } - Ok(()) -} - -fn recover_file( - file: &mut File, - namespace: ShadowWalNamespaceV1, -) -> Result { - let original_len = file.metadata()?.len(); - if original_len < FILE_HEADER_LEN as u64 { - return Err(ShadowWalErrorV1::TruncatedFileHeader { - actual: original_len, - expected: FILE_HEADER_LEN, - }); - } - file.seek(SeekFrom::Start(0))?; - let mut file_header = [0u8; FILE_HEADER_LEN]; - file.read_exact(&mut file_header)?; - validate_file_header(&file_header, namespace)?; - - let mut batches = Vec::new(); - let mut record_count = 0u64; - let mut offset = FILE_HEADER_LEN as u64; - let mut expected_sequence = 0u64; - while offset < original_len { - let remaining = original_len - offset; - if remaining < FRAME_HEADER_LEN as u64 { - break; - } - - file.seek(SeekFrom::Start(offset))?; - let mut frame_header = [0u8; FRAME_HEADER_LEN]; - file.read_exact(&mut frame_header)?; - validate_frame_header(&frame_header, offset)?; - let sequence = read_u64(&frame_header[8..16]); - if sequence != expected_sequence { - return Err(ShadowWalErrorV1::UnexpectedFrameSequence { - offset, - expected: expected_sequence, - actual: sequence, - }); - } - let payload_len_u64 = read_u64(&frame_header[16..24]); - let payload_len = - usize::try_from(payload_len_u64).map_err(|_| ShadowWalErrorV1::CorruptFrame { - offset, - reason: "payload length does not fit usize", - })?; - if payload_len > MAX_SHADOW_WAL_FRAME_PAYLOAD_V1 { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "payload exceeds configured maximum", - }); - } - let frame_len = (FRAME_HEADER_LEN as u64) - .checked_add(payload_len_u64) - .and_then(|length| length.checked_add(FRAME_TRAILER_LEN as u64)) - .ok_or(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "frame length overflow", - })?; - if frame_len > remaining { - break; - } - - let mut payload = vec![0u8; payload_len]; - file.read_exact(&mut payload)?; - let mut trailer = [0u8; FRAME_TRAILER_LEN]; - file.read_exact(&mut trailer)?; - if read_u32(&trailer[4..8]) != FRAME_COMMIT_MAGIC { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "invalid frame commit marker", - }); - } - let mut checksum_bytes = Vec::with_capacity(FRAME_HEADER_LEN + payload.len()); - checksum_bytes.extend_from_slice(&frame_header); - checksum_bytes.extend_from_slice(&payload); - if read_u32(&trailer[0..4]) != crc32c(&checksum_bytes) { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "frame checksum mismatch", - }); - } - - let declared_records = read_u32(&frame_header[24..28]); - let records = decode_records(&payload, declared_records, offset)?; - record_count = record_count - .checked_add(u64::from(declared_records)) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - let end_offset = offset + frame_len; - batches.push(RecoveredBatchV1 { - sequence, - start_offset: offset, - end_offset, - records, - }); - offset = end_offset; - expected_sequence = expected_sequence - .checked_add(1) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - } - - let discarded_tail_bytes = original_len - offset; - if discarded_tail_bytes != 0 { - file.set_len(offset)?; - file.sync_all()?; - } - Ok(ShadowWalRecoveryV1 { - batches, - durable_file_len: offset, - record_count, - discarded_tail_bytes, - }) -} - -fn encode_frame(sequence: u64, records: &[Vec]) -> Result, ShadowWalErrorV1> { - if records.is_empty() { - return Err(ShadowWalErrorV1::EmptyBatch); - } - if records.len() > MAX_SHADOW_WAL_BATCH_RECORDS_V1 { - return Err(ShadowWalErrorV1::TooManyRecords(records.len())); - } - let mut payload_len = 0usize; - for record in records { - if record.len() > MAX_SHADOW_WAL_RECORD_SIZE_V1 { - return Err(ShadowWalErrorV1::RecordTooLarge(record.len())); - } - payload_len = payload_len - .checked_add(4) - .and_then(|length| length.checked_add(record.len())) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - } - if payload_len > MAX_SHADOW_WAL_FRAME_PAYLOAD_V1 { - return Err(ShadowWalErrorV1::FramePayloadTooLarge(payload_len)); - } - let payload_len_u64 = - u64::try_from(payload_len).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; - let record_count = - u32::try_from(records.len()).map_err(|_| ShadowWalErrorV1::LengthOverflow)?; - - let total_len = FRAME_HEADER_LEN - .checked_add(payload_len) - .and_then(|length| length.checked_add(FRAME_TRAILER_LEN)) - .ok_or(ShadowWalErrorV1::LengthOverflow)?; - let mut frame = Vec::with_capacity(total_len); - frame.extend_from_slice(&FRAME_MAGIC.to_be_bytes()); - frame.extend_from_slice(&SHADOW_WAL_FORMAT_VERSION_V1.to_be_bytes()); - frame.extend_from_slice(&0u16.to_be_bytes()); - frame.extend_from_slice(&sequence.to_be_bytes()); - frame.extend_from_slice(&payload_len_u64.to_be_bytes()); - frame.extend_from_slice(&record_count.to_be_bytes()); - frame.extend_from_slice(&0u32.to_be_bytes()); - let header_checksum = crc32c(&frame); - frame.extend_from_slice(&header_checksum.to_be_bytes()); - debug_assert_eq!(frame.len(), FRAME_HEADER_LEN); - for record in records { - let record_len = - u32::try_from(record.len()).expect("record length was validated before encoding"); - frame.extend_from_slice(&record_len.to_be_bytes()); - frame.extend_from_slice(record); - } - let frame_checksum = crc32c(&frame); - frame.extend_from_slice(&frame_checksum.to_be_bytes()); - frame.extend_from_slice(&FRAME_COMMIT_MAGIC.to_be_bytes()); - debug_assert_eq!(frame.len(), total_len); - Ok(frame) -} - -fn validate_frame_header( - header: &[u8; FRAME_HEADER_LEN], - offset: u64, -) -> Result<(), ShadowWalErrorV1> { - if read_u32(&header[0..4]) != FRAME_MAGIC { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "invalid frame magic", - }); - } - if read_u16(&header[4..6]) != SHADOW_WAL_FORMAT_VERSION_V1 { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "unsupported frame version", - }); - } - if read_u16(&header[6..8]) != 0 || read_u32(&header[28..32]) != 0 { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "nonzero frame flags", - }); - } - if read_u32(&header[32..36]) != crc32c(&header[..FRAME_HEADER_PREFIX_LEN]) { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "frame header checksum mismatch", - }); - } - let record_count = read_u32(&header[24..28]); - if record_count == 0 - || usize::try_from(record_count).expect("u32 always fits supported usize") - > MAX_SHADOW_WAL_BATCH_RECORDS_V1 - { - return Err(ShadowWalErrorV1::CorruptFrame { - offset, - reason: "invalid frame record count", - }); - } - Ok(()) -} - -fn decode_records( - payload: &[u8], - declared_records: u32, - frame_offset: u64, -) -> Result>, ShadowWalErrorV1> { - let declared_records = - usize::try_from(declared_records).expect("u32 always fits supported usize"); - let mut records = Vec::with_capacity(declared_records); - let mut cursor = 0usize; - for _ in 0..declared_records { - let length_end = cursor - .checked_add(4) - .ok_or(ShadowWalErrorV1::CorruptFrame { - offset: frame_offset, - reason: "record length offset overflow", - })?; - let Some(length_bytes) = payload.get(cursor..length_end) else { - return Err(ShadowWalErrorV1::CorruptFrame { - offset: frame_offset, - reason: "truncated record length", - }); - }; - let record_len = - usize::try_from(read_u32(length_bytes)).expect("u32 always fits supported usize"); - if record_len > MAX_SHADOW_WAL_RECORD_SIZE_V1 { - return Err(ShadowWalErrorV1::CorruptFrame { - offset: frame_offset, - reason: "record exceeds configured maximum", - }); - } - let record_end = - length_end - .checked_add(record_len) - .ok_or(ShadowWalErrorV1::CorruptFrame { - offset: frame_offset, - reason: "record offset overflow", - })?; - let Some(record) = payload.get(length_end..record_end) else { - return Err(ShadowWalErrorV1::CorruptFrame { - offset: frame_offset, - reason: "truncated record", - }); - }; - records.push(record.to_vec()); - cursor = record_end; - } - if cursor != payload.len() { - return Err(ShadowWalErrorV1::CorruptFrame { - offset: frame_offset, - reason: "trailing frame payload bytes", - }); - } - Ok(records) -} - -fn sync_parent_directory(path: &Path) -> io::Result<()> { - let parent = nonempty_parent(path).unwrap_or_else(|| Path::new(".")); - File::open(parent)?.sync_all() -} - -/// Create a potentially nested WAL parent and durably publish every new -/// directory name from the nearest existing ancestor downwards. -fn create_parent_directories_durable(parent: &Path) -> io::Result<()> { - let mut missing = Vec::new(); - let mut cursor = parent.to_path_buf(); - loop { - if cursor.as_os_str().is_empty() { - cursor = PathBuf::from("."); - } - match fs::symlink_metadata(&cursor) { - Ok(_) => break, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - missing.push(cursor.clone()); - cursor = nonempty_parent(&cursor) - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from(".")); - } - Err(error) => return Err(error), - } - } - - fs::create_dir_all(parent)?; - for directory in missing.iter().rev() { - File::open(directory)?.sync_all()?; - sync_parent_directory(directory)?; - } - Ok(()) -} - -fn nonempty_parent(path: &Path) -> Option<&Path> { - path.parent() - .filter(|parent| !parent.as_os_str().is_empty()) -} - -fn read_u16(bytes: &[u8]) -> u16 { - u16::from_be_bytes(bytes.try_into().expect("fixed u16 range")) -} - -fn read_u32(bytes: &[u8]) -> u32 { - u32::from_be_bytes(bytes.try_into().expect("fixed u32 range")) -} - -fn read_u64(bytes: &[u8]) -> u64 { - u64::from_be_bytes(bytes.try_into().expect("fixed u64 range")) -} - -// Table-free CRC32C (Castagnoli). WAL writes are not on the protocol hot path, -// and avoiding another dependency keeps this isolated adapter self-contained. -fn crc32c(bytes: &[u8]) -> u32 { - let mut crc = !0u32; - for byte in bytes { - crc ^= u32::from(*byte); - for _ in 0..8 { - let mask = 0u32.wrapping_sub(crc & 1); - crc = (crc >> 1) ^ (0x82F6_3B78 & mask); - } - } - !crc -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - committee::Committee, starfish_rbc_dag::RbcDagProtocolInstanceId, - types::BlockAuthenticationScheme, - }; - use std::io::{Seek, SeekFrom, Write}; - - fn namespace_with_stakes( - instance_marker: u8, - own_authority: AuthorityIndex, - stakes: Vec, - ) -> ShadowWalNamespaceV1 { - let committee = Committee::new_test(stakes); - let context = RbcDagContextV1::new( - RbcDagProtocolInstanceId::new([instance_marker; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ) - .unwrap(); - ShadowWalNamespaceV1::new(context, own_authority) - } - - fn namespace(instance_marker: u8, own_authority: AuthorityIndex) -> ShadowWalNamespaceV1 { - namespace_with_stakes(instance_marker, own_authority, vec![1; 4]) - } - - fn wal_path(directory: &tempfile::TempDir) -> PathBuf { - directory.path().join("shadow").join("rbc-dag.wal") - } - - fn append_raw(path: &Path, bytes: &[u8]) { - let mut file = OpenOptions::new().append(true).open(path).unwrap(); - file.write_all(bytes).unwrap(); - file.sync_all().unwrap(); - } - - fn overwrite_byte(path: &Path, offset: u64) { - let mut file = OpenOptions::new() - .read(true) - .write(true) - .open(path) - .unwrap(); - file.seek(SeekFrom::Start(offset)).unwrap(); - let mut byte = [0u8; 1]; - file.read_exact(&mut byte).unwrap(); - byte[0] ^= 0x80; - file.seek(SeekFrom::Start(offset)).unwrap(); - file.write_all(&byte).unwrap(); - file.sync_all().unwrap(); - } - - #[test] - fn crc32c_matches_the_standard_check_vector() { - assert_eq!(crc32c(b"123456789"), 0xE306_9283); - } - - #[test] - fn batches_reopen_with_exact_record_bytes_and_continue_sequence() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA1, 2); - let (mut wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.batch_count(), 0); - assert_eq!(recovery.durable_file_len(), FILE_HEADER_LEN as u64); - - let first_records = vec![b"first".to_vec(), Vec::new(), vec![0, 0xFF, 7]]; - let first = wal.append_batch(&first_records).unwrap(); - assert_eq!(first.sequence(), 0); - assert_eq!(first.start_offset(), FILE_HEADER_LEN as u64); - assert_eq!(first.record_count(), 3); - let second_records = vec![b"second".to_vec()]; - let second = wal.append_batch(&second_records).unwrap(); - assert_eq!(second.sequence(), 1); - assert_eq!(second.start_offset(), first.end_offset()); - let summary = wal.shutdown().unwrap(); - assert_eq!(summary.batch_count(), 2); - assert_eq!(summary.record_count(), 4); - - let (mut reopened, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.batch_count(), 2); - assert_eq!(recovery.record_count(), 4); - assert_eq!( - recovery.records(), - vec![ - b"first".as_slice(), - b"".as_slice(), - [0, 0xFF, 7].as_slice(), - b"second".as_slice(), - ] - ); - assert_eq!(recovery.durable_file_len(), summary.file_len()); - assert_eq!( - reopened - .append_batch(&[b"third".to_vec()]) - .unwrap() - .sequence(), - 2 - ); - reopened.shutdown().unwrap(); - - let (_, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.batch_count(), 3); - assert_eq!( - recovery.records().last().copied(), - Some(b"third".as_slice()) - ); - } - - #[test] - fn every_incomplete_final_frame_byte_boundary_is_discarded_once() { - let torn = encode_frame(1, &[b"not-durable".to_vec()]).unwrap(); - for cut in 1..torn.len() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA2, 1); - let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - wal.append_batch(&[b"durable".to_vec()]).unwrap(); - let durable_len = wal.file_len(); - wal.shutdown().unwrap(); - - append_raw(&path, &torn[..cut]); - let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.records(), vec![b"durable".as_slice()]); - assert_eq!(recovery.discarded_tail_bytes(), cut as u64); - assert_eq!(wal.file_len(), durable_len); - wal.shutdown().unwrap(); - - let (_, clean_recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(clean_recovery.discarded_tail_bytes(), 0); - } - } - - #[test] - fn partial_payload_or_trailer_is_discarded_as_one_torn_final_batch() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA3, 0); - let (wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - wal.shutdown().unwrap(); - let frame = encode_frame(0, &[vec![0xAB; 128], b"tail".to_vec()]).unwrap(); - - for cut in [FRAME_HEADER_LEN + 10, frame.len() - 1] { - let original = fs::read(&path).unwrap(); - append_raw(&path, &frame[..cut]); - let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.batch_count(), 0); - assert_eq!(recovery.discarded_tail_bytes(), cut as u64); - wal.shutdown().unwrap(); - fs::write(&path, &original).unwrap(); - } - } - - #[test] - fn checksum_corruption_in_complete_first_or_last_frame_is_rejected() { - for corrupt_first in [true, false] { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA4, 3); - let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - let first = wal.append_batch(&[b"one".to_vec()]).unwrap(); - let second = wal.append_batch(&[b"two".to_vec()]).unwrap(); - wal.shutdown().unwrap(); - let position = if corrupt_first { first } else { second }; - overwrite_byte(&path, position.start_offset() + FRAME_HEADER_LEN as u64 + 4); - assert!(matches!( - ShadowWalV1::open(&path, namespace), - Err(ShadowWalErrorV1::CorruptFrame { - reason: "frame checksum mismatch", - .. - }) - )); - } - } - - #[test] - fn corrupted_frame_header_is_not_misclassified_as_a_torn_tail() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA5, 1); - let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - let frame = wal.append_batch(&[b"record".to_vec()]).unwrap(); - wal.shutdown().unwrap(); - overwrite_byte(&path, frame.start_offset() + 18); - assert!(matches!( - ShadowWalV1::open(&path, namespace), - Err(ShadowWalErrorV1::CorruptFrame { - reason: "frame header checksum mismatch", - .. - }) - )); - } - - #[test] - fn namespace_is_bound_to_instance_committee_authority_and_version() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let expected_namespace = namespace(0xA6, 1); - let (wal, _) = ShadowWalV1::open(&path, expected_namespace).unwrap(); - wal.shutdown().unwrap(); - - assert!(matches!( - ShadowWalV1::open(&path, namespace(0xA7, 1)), - Err(ShadowWalErrorV1::NamespaceMismatch) - )); - assert!(matches!( - ShadowWalV1::open(&path, namespace(0xA6, 2)), - Err(ShadowWalErrorV1::NamespaceMismatch) - )); - assert!(matches!( - ShadowWalV1::open(&path, namespace_with_stakes(0xA6, 1, vec![1, 1, 1, 2])), - Err(ShadowWalErrorV1::NamespaceMismatch) - )); - - let version_offset = 16u64; - overwrite_byte(&path, version_offset + 1); - assert!(matches!( - ShadowWalV1::open(&path, expected_namespace), - Err(ShadowWalErrorV1::UnsupportedFileVersion(_)) - )); - } - - #[test] - fn truncated_or_corrupt_file_header_is_rejected_without_reinitializing() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA8, 0); - let (wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - wal.shutdown().unwrap(); - OpenOptions::new() - .write(true) - .open(&path) - .unwrap() - .set_len((FILE_HEADER_LEN - 1) as u64) - .unwrap(); - assert!(matches!( - ShadowWalV1::open(&path, namespace), - Err(ShadowWalErrorV1::TruncatedFileHeader { .. }) - )); - - fs::write(&path, encode_file_header(namespace)).unwrap(); - overwrite_byte(&path, 30); - assert!(matches!( - ShadowWalV1::open(&path, namespace), - Err(ShadowWalErrorV1::InvalidFileHeaderChecksum) - )); - } - - #[test] - fn preexisting_empty_file_is_rejected_instead_of_erasing_durable_identity() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - File::create(&path).unwrap().sync_all().unwrap(); - - assert!(matches!( - ShadowWalV1::open(&path, namespace(0xAB, 0)), - Err(ShadowWalErrorV1::TruncatedFileHeader { - actual: 0, - expected: FILE_HEADER_LEN, - }) - )); - assert_eq!(fs::metadata(path).unwrap().len(), 0); - } - - #[cfg(unix)] - #[test] - fn preexisting_dangling_symlink_fails_closed_without_publication_retry() { - use std::os::unix::fs::symlink; - - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - symlink("missing-shadow-wal", &path).unwrap(); - - assert!(matches!( - ShadowWalV1::open(&path, namespace(0xAF, 0)), - Err(ShadowWalErrorV1::Io(error)) - if error.kind() == io::ErrorKind::NotFound - )); - assert!( - fs::symlink_metadata(&path) - .unwrap() - .file_type() - .is_symlink() - ); - } - - #[test] - fn crash_before_initialization_publish_never_exposes_a_partial_header() { - for cut in [0, 1, FILE_HEADER_LEN - 1, FILE_HEADER_LEN] { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xAC, 0); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - - // Model a process dying at any write boundary before the atomic - // publish. Its private same-directory file may survive, but the - // canonical name must remain absent and independently creatable. - let (temporary_path, mut temporary) = create_initialization_temp(&path).unwrap(); - temporary - .write_all(&encode_file_header(namespace)[..cut]) - .unwrap(); - temporary.sync_all().unwrap(); - drop(temporary); - assert!(!path.exists()); - - let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.durable_file_len(), FILE_HEADER_LEN as u64); - assert_eq!(recovery.batch_count(), 0); - wal.shutdown().unwrap(); - assert_eq!(fs::metadata(&path).unwrap().len(), FILE_HEADER_LEN as u64); - - // Orphan cleanup is opportunistic and never part of identifying - // the canonical WAL. Remove the simulated crashed process's file - // so the test itself leaves no debris. - assert!(temporary_path.exists()); - fs::remove_file(temporary_path).unwrap(); - } - } - - #[test] - fn initialization_race_never_replaces_an_existing_malformed_target() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xAD, 0); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - - let (temporary_path, mut temporary) = create_initialization_temp(&path).unwrap(); - temporary.write_all(&encode_file_header(namespace)).unwrap(); - temporary.sync_all().unwrap(); - File::create(&path).unwrap().sync_all().unwrap(); - - let error = atomic_rename_noreplace(&temporary_path, &path).unwrap_err(); - assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); - assert_eq!(fs::metadata(&path).unwrap().len(), 0); - assert!(temporary_path.exists()); - drop(temporary); - fs::remove_file(temporary_path).unwrap(); - - assert!(matches!( - ShadowWalV1::open(&path, namespace), - Err(ShadowWalErrorV1::TruncatedFileHeader { - actual: 0, - expected: FILE_HEADER_LEN, - }) - )); - assert_eq!(fs::metadata(path).unwrap().len(), 0); - } - - #[test] - fn successful_initialization_publishes_only_the_canonical_name() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xAE, 0); - let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.durable_file_len(), FILE_HEADER_LEN as u64); - wal.shutdown().unwrap(); - - let entries = fs::read_dir(path.parent().unwrap()) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect::>(); - assert_eq!(entries, vec![path.file_name().unwrap()]); - - let (wal, reopened) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(reopened.discarded_tail_bytes(), 0); - assert_eq!(reopened.durable_file_len(), FILE_HEADER_LEN as u64); - wal.shutdown().unwrap(); - } - - #[test] - fn invalid_batch_is_rejected_without_changing_the_file() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xA9, 0); - let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - let initial_len = wal.file_len(); - assert!(matches!( - wal.append_batch(&[]), - Err(ShadowWalErrorV1::EmptyBatch) - )); - assert!(matches!( - wal.append_batch(&[vec![0; MAX_SHADOW_WAL_RECORD_SIZE_V1 + 1]]), - Err(ShadowWalErrorV1::RecordTooLarge(_)) - )); - assert_eq!(wal.file_len(), initial_len); - assert_eq!(wal.batch_count(), 0); - assert!(!wal.is_poisoned()); - wal.shutdown().unwrap(); - } - - #[test] - fn external_file_mutation_poisoning_requires_reopen() { - let directory = tempfile::tempdir().unwrap(); - let path = wal_path(&directory); - let namespace = namespace(0xAA, 1); - let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - append_raw(&path, &[0xDE, 0xAD]); - assert!(matches!( - wal.append_batch(&[b"event".to_vec()]), - Err(ShadowWalErrorV1::ExternalFileMutation { .. }) - )); - assert!(wal.is_poisoned()); - assert!(matches!( - wal.append_batch(&[b"again".to_vec()]), - Err(ShadowWalErrorV1::Poisoned) - )); - drop(wal); - - let (mut reopened, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - assert_eq!(recovery.discarded_tail_bytes(), 2); - reopened.append_batch(&[b"after-reopen".to_vec()]).unwrap(); - reopened.shutdown().unwrap(); - } -} diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow.rs deleted file mode 100644 index ba10e87f..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow.rs +++ /dev/null @@ -1,2659 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Durable, single-owner shadow execution for the embedded-RBC Starfish DAG. -//! -//! This adapter is deliberately non-authoritative: it consumes the same -//! carrier bytes as the live protocol, persists its own deterministic input -//! and trace log, and reports comparison results without influencing the live -//! protocol. One [`ShadowWalV1`] batch is one reducer transition. Effects are -//! returned only after that complete batch has reached durable storage. - -use std::{ - collections::{BTreeMap, BTreeSet}, - error::Error, - fmt, - path::Path, -}; - -use crate::{ - crypto::{MacKey, MlDsa44Signer, MlDsa65Signer, Signer, TransactionsCommitment}, - starfish_rbc_dag::{ - AuthenticatedCarrierV1, CandidateCarrierV1, CarrierAuthenticationV1, CarrierAuthorizerV1, - CarrierHeaderV1Args, LocallyAuthenticatedCarrierV1, RbcDagCommitteeContextV1, - RbcDagContextV1, RbcDagError, RbcPhaseStatementV1, - journal::{ - IngressProvenanceV1, JournalErrorV1, JournalEventV1, ValidatedJournalBatchV1, - WriteAheadJournalV1, - }, - model::{ModelEffect, ModelError, ModelInputRecord, ModelTraceEvent, RbcDagModel}, - storage::{ - MAX_SHADOW_WAL_RECORD_SIZE_V1, ShadowWalErrorV1, ShadowWalNamespaceV1, - ShadowWalSummaryV1, ShadowWalV1, - }, - }, - types::{ - AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, - RoundNumber, TimestampNs, - }, -}; - -const RAW_RECORD_MAGIC: &[u8; 4] = b"SRD3"; -const RAW_RECORD_VERSION_V1: u8 = 1; -const RAW_RECORD_HEADER_SIZE: usize = 80; - -const RECORD_AUTHENTICATED_INGRESS: u8 = 0x01; -const RECORD_CANDIDATE_RETENTION: u8 = 0x02; -const RECORD_CANDIDATE_RECOVERY: u8 = 0x03; -const RECORD_LOCAL_OUTBOUND_CONTENT: u8 = 0x04; -const RECORD_MODEL_TRACE: u8 = 0x10; -const RECORD_LOCAL_OUTBOUND_SIDECAR: u8 = 0x11; -const RECORD_LOCAL_OUTBOUND_EXPOSE: u8 = 0x12; - -const TRACE_ADMISSION_LOCKED: u8 = 0x00; -const TRACE_LOCAL_PHASE_LOCKED: u8 = 0x01; -const TRACE_PHASE_ENTRY_APPLIED: u8 = 0x02; -const TRACE_PHASE_CURSOR_ADVANCED: u8 = 0x03; -const TRACE_LOCAL_CARRIER_FIXED: u8 = 0x04; -const TRACE_DELIVERY_LOCKED: u8 = 0x05; -const TRACE_EFFECT: u8 = 0x06; - -const EFFECT_NEED_CARRIER: u8 = 0x00; -const EFFECT_DELIVERED: u8 = 0x01; -const EFFECT_PREFIX_ADVANCED: u8 = 0x02; -const EFFECT_CARRIER_ROUND_ADVANCED: u8 = 0x03; - -const PHASE_ECHO: u8 = 0x00; -const PHASE_READY: u8 = 0x01; -const PROVENANCE_DIRECT: u8 = 0x00; -const PROVENANCE_RELAYED: u8 = 0x01; - -/// Shadow-benchmark-only resource guard for newly arriving, unsolicited -/// values. This is not a protocol-safe pruning rule: asynchronous delivery -/// can delay an honest INIT by more than this many rounds. A production -/// protocol must derive pruning from a certified/committed watermark instead. -/// Exact RBC recovery requests are exempt from this prototype guard. -const SHADOW_BENCHMARK_UNSOLICITED_RETENTION_WINDOW_ROUNDS_V1: RoundNumber = 64; - -/// Local authentication material owned by exactly one shadow core. -/// -/// The MAC variant contains the local authority's complete pairwise keyring: -/// it creates full outbound vectors and verifies this receiver's inbound tag. -#[derive(Clone, Debug)] -pub(crate) enum ShadowAuthorizerV1 { - Ed25519(Signer), - MlDsa44(MlDsa44Signer), - MlDsa65(MlDsa65Signer), - MacVector(Vec), -} - -impl ShadowAuthorizerV1 { - fn scheme(&self) -> BlockAuthenticationScheme { - match self { - Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, - Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, - Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, - Self::MacVector(_) => BlockAuthenticationScheme::MacVector, - } - } - - fn authorizer(&self, authority: AuthorityIndex) -> CarrierAuthorizerV1<'_> { - match self { - Self::Ed25519(signer) => CarrierAuthorizerV1::Ed25519 { authority, signer }, - Self::MlDsa44(signer) => CarrierAuthorizerV1::MlDsa44 { authority, signer }, - Self::MlDsa65(signer) => CarrierAuthorizerV1::MlDsa65 { authority, signer }, - Self::MacVector(keys) => CarrierAuthorizerV1::MacVector { authority, keys }, - } - } - - fn inbound_mac_keys(&self) -> &[MacKey] { - match self { - Self::MacVector(keys) => keys, - Self::Ed25519(_) | Self::MlDsa44(_) | Self::MlDsa65(_) => &[], - } - } -} - -/// Exact, peer-independent bytes retained for first send and retransmission. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct ShadowOutboundEnvelopeV1 { - reference: BlockReference, - canonical_carrier_wire: Vec, - authentication_sidecar: Vec, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ShadowIngressDispositionV1 { - Authenticated, - CandidateRetained, - IgnoredDuplicateConflictOrStale, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct ShadowIngressOutcomeV1 { - disposition: ShadowIngressDispositionV1, - effects: Vec, -} - -impl ShadowIngressOutcomeV1 { - pub(crate) fn disposition(&self) -> ShadowIngressDispositionV1 { - self.disposition - } - - pub(crate) fn effects(&self) -> &[ModelEffect] { - &self.effects - } - - fn new(disposition: ShadowIngressDispositionV1, effects: Vec) -> Self { - Self { - disposition, - effects, - } - } -} - -impl ShadowOutboundEnvelopeV1 { - pub(crate) fn reference(&self) -> BlockReference { - self.reference - } - - pub(crate) fn canonical_carrier_wire(&self) -> &[u8] { - &self.canonical_carrier_wire - } - - pub(crate) fn authentication_sidecar(&self) -> &[u8] { - &self.authentication_sidecar - } -} - -/// Protocol-independent delivery identity used for direct/shadow comparison. -/// -/// A block reference is intentionally absent: direct and shadow protocols may -/// commit different headers for the same transaction payload. -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) struct ShadowDeliveryIdentityV1 { - pub(crate) author: AuthorityIndex, - pub(crate) round: RoundNumber, - pub(crate) transactions_commitment: TransactionsCommitment, -} - -impl ShadowDeliveryIdentityV1 { - pub(crate) const fn new( - author: AuthorityIndex, - round: RoundNumber, - transactions_commitment: TransactionsCommitment, - ) -> Self { - Self { - author, - round, - transactions_commitment, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) struct ShadowDeliverySlotV1 { - pub(crate) author: AuthorityIndex, - pub(crate) round: RoundNumber, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum ShadowDeliveryComparisonV1 { - Match, - Mismatch { - direct_only: Vec, - shadow_only: Vec, - }, - Ambiguous { - slots: Vec, - }, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct ShadowOpenReportV1 { - replayed_batches: u64, - discarded_tail_bytes: u64, - recovery_effects: Vec, -} - -impl ShadowOpenReportV1 { - pub(crate) fn replayed_batches(&self) -> u64 { - self.replayed_batches - } - - pub(crate) fn discarded_tail_bytes(&self) -> u64 { - self.discarded_tail_bytes - } - - /// Final outstanding recovery requests only. Historical delivery, clock, - /// and already-satisfied recovery effects are never reissued on restart. - pub(crate) fn recovery_effects(&self) -> &[ModelEffect] { - &self.recovery_effects - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum ShadowCodecErrorV1 { - UnexpectedEnd, - InvalidMagic, - UnsupportedVersion(u8), - InvalidFlags(u16), - ContextMismatch, - AuthorityMismatch { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - AuthenticationSchemeMismatch, - UnknownRecordKind(u8), - InvalidRecordLength(usize), - TrailingBytes(usize), - InvalidProvenance(u8), - InvalidPhase(u8), - InvalidTrace(u8), - InvalidEffect(u8), - NonCanonicalHolders, - LengthOverflow, -} - -impl fmt::Display for ShadowCodecErrorV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "Starfish-RBC-DAG shadow codec error: {self:?}") - } -} - -impl Error for ShadowCodecErrorV1 {} - -#[derive(Debug)] -pub(crate) enum ShadowErrorV1 { - Wal(ShadowWalErrorV1), - Codec(ShadowCodecErrorV1), - Carrier(RbcDagError), - Model(ModelError), - Journal(JournalErrorV1), - ContextMismatch, - UnknownAuthority(AuthorityIndex), - NonCanonicalProvenance, - AuthorizerSchemeMismatch, - AuthorizerKeyMismatch, - InvalidAuthorizerKeyringLength { - expected: usize, - actual: usize, - }, - InvalidBatch(&'static str), - NonCanonicalCarrier, - NonCanonicalAuthentication, - TraceMismatch { - batch_sequence: u64, - }, - ReplayPolicyViolation { - batch_sequence: u64, - reason: &'static str, - }, - UnrequestedRecovery(BlockReference), - SlotCandidateLimit { - author: AuthorityIndex, - round: RoundNumber, - limit: usize, - }, - MissingOutboundCandidate(BlockReference), - MissingDeliveredCandidate(BlockReference), - PostDurabilityCommit(ModelError), - PostDurabilityJournal(JournalErrorV1), - Poisoned, -} - -impl fmt::Display for ShadowErrorV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Wal(error) => write!(formatter, "{error}"), - Self::Codec(error) => write!(formatter, "{error}"), - Self::Carrier(error) => write!(formatter, "{error}"), - Self::Model(error) => write!(formatter, "{error}"), - Self::Journal(error) => write!(formatter, "{error}"), - Self::ContextMismatch => formatter.write_str("shadow protocol context mismatch"), - Self::UnknownAuthority(authority) => { - write!(formatter, "unknown shadow authority {authority}") - } - Self::NonCanonicalProvenance => formatter.write_str( - "non-canonical shadow ingress provenance: an author's own peer must be direct", - ), - Self::AuthorizerSchemeMismatch => { - formatter.write_str("shadow authorizer scheme mismatch") - } - Self::AuthorizerKeyMismatch => formatter.write_str("shadow authorizer key mismatch"), - Self::InvalidAuthorizerKeyringLength { expected, actual } => write!( - formatter, - "invalid shadow authorizer keyring length: expected {expected}, got {actual}" - ), - Self::InvalidBatch(reason) => write!(formatter, "invalid shadow WAL batch: {reason}"), - Self::NonCanonicalCarrier => { - formatter.write_str("non-canonical shadow carrier encoding") - } - Self::NonCanonicalAuthentication => { - formatter.write_str("non-canonical shadow authentication encoding") - } - Self::TraceMismatch { batch_sequence } => write!( - formatter, - "shadow trace mismatch in WAL batch {batch_sequence}" - ), - Self::ReplayPolicyViolation { - batch_sequence, - reason, - } => write!( - formatter, - "shadow replay policy violation in WAL batch {batch_sequence}: {reason}" - ), - Self::UnrequestedRecovery(reference) => { - write!(formatter, "unrequested shadow recovery for {reference}") - } - Self::SlotCandidateLimit { - author, - round, - limit, - } => write!( - formatter, - "shadow candidate limit {limit} reached for slot ({author}, {round})" - ), - Self::MissingOutboundCandidate(reference) => { - write!( - formatter, - "missing persisted outbound shadow candidate {reference}" - ) - } - Self::MissingDeliveredCandidate(reference) => { - write!(formatter, "missing delivered shadow candidate {reference}") - } - Self::PostDurabilityCommit(error) => write!( - formatter, - "shadow model commit failed after WAL durability: {error}" - ), - Self::PostDurabilityJournal(error) => write!( - formatter, - "shadow journal commit failed after WAL durability: {error}" - ), - Self::Poisoned => formatter.write_str("shadow core is poisoned"), - } - } -} - -impl Error for ShadowErrorV1 { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Wal(error) => Some(error), - Self::Codec(error) => Some(error), - Self::Carrier(error) => Some(error), - Self::Model(error) | Self::PostDurabilityCommit(error) => Some(error), - Self::Journal(error) | Self::PostDurabilityJournal(error) => Some(error), - _ => None, - } - } -} - -impl From for ShadowErrorV1 { - fn from(error: ShadowWalErrorV1) -> Self { - Self::Wal(error) - } -} - -impl From for ShadowErrorV1 { - fn from(error: ShadowCodecErrorV1) -> Self { - Self::Codec(error) - } -} - -impl From for ShadowErrorV1 { - fn from(error: RbcDagError) -> Self { - Self::Carrier(error) - } -} - -impl From for ShadowErrorV1 { - fn from(error: ModelError) -> Self { - Self::Model(error) - } -} - -impl From for ShadowErrorV1 { - fn from(error: JournalErrorV1) -> Self { - Self::Journal(error) - } -} - -#[derive(Clone)] -enum ShadowInputV1 { - AuthenticatedIngress { - authenticated: AuthenticatedCarrierV1, - provenance: IngressProvenanceV1, - }, - CandidateRetention(CandidateCarrierV1), - CandidateRecovery(CandidateCarrierV1), - LocalOutbound(LocallyAuthenticatedCarrierV1), -} - -impl ShadowInputV1 { - fn model_input(&self) -> ModelInputRecord { - match self { - Self::AuthenticatedIngress { authenticated, .. } => { - ModelInputRecord::AuthenticatedIngress(authenticated.clone()) - } - Self::CandidateRetention(candidate) => { - ModelInputRecord::CandidateRetained(candidate.clone()) - } - Self::CandidateRecovery(candidate) => { - ModelInputRecord::CandidateRecovered(candidate.clone()) - } - Self::LocalOutbound(authenticated) => { - ModelInputRecord::LocalCarrierFixed(authenticated.clone()) - } - } - } - - fn candidate(&self) -> &CandidateCarrierV1 { - match self { - Self::AuthenticatedIngress { authenticated, .. } => authenticated.candidate(), - Self::CandidateRetention(candidate) | Self::CandidateRecovery(candidate) => candidate, - Self::LocalOutbound(authenticated) => authenticated.candidate(), - } - } - - fn is_local(&self) -> bool { - matches!(self, Self::LocalOutbound(_)) - } -} - -struct DecodedRawRecord { - kind: u8, - payload: Vec, -} - -/// Synchronous, non-authoritative shadow core. -/// -/// This type has one mutable model, journal, and WAL handle and intentionally -/// offers no shared-state wrapper. A caller may move it between threads but -/// must preserve exclusive ownership. -pub(crate) struct StarfishRbcDagShadowV1 { - committee: RbcDagCommitteeContextV1, - context: RbcDagContextV1, - own_authority: AuthorityIndex, - authorizer: ShadowAuthorizerV1, - model: RbcDagModel, - journal: WriteAheadJournalV1, - wal: ShadowWalV1, - candidates: BTreeMap, - delivered: BTreeSet, - authenticated_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, - ordinarily_retained_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, - slot_candidates: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, - requested_recoveries: BTreeMap>, - poisoned: bool, -} - -impl StarfishRbcDagShadowV1 { - pub(crate) fn open( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - ) -> Result<(Self, ShadowOpenReportV1), ShadowErrorV1> { - validate_configuration(&committee, own_authority, context, &authorizer)?; - let namespace = ShadowWalNamespaceV1::new(context, own_authority); - let (wal, recovery) = ShadowWalV1::open(path, namespace)?; - let replayed_batches = recovery.batch_count(); - let discarded_tail_bytes = recovery.discarded_tail_bytes(); - - let model = RbcDagModel::new(committee.committee_arc(), own_authority, context)?; - let journal = WriteAheadJournalV1::new(context, own_authority); - let mut core = Self { - committee, - context, - own_authority, - authorizer, - model, - journal, - wal, - candidates: BTreeMap::new(), - delivered: BTreeSet::new(), - authenticated_slots: BTreeMap::new(), - ordinarily_retained_slots: BTreeMap::new(), - slot_candidates: BTreeMap::new(), - requested_recoveries: BTreeMap::new(), - poisoned: false, - }; - - for batch in recovery.batches() { - let input = core.decode_batch(batch.records())?; - core.apply_replayed(input, batch.records(), batch.sequence())?; - } - let recovery_effects = core - .requested_recoveries - .iter() - .map(|(target, holders)| ModelEffect::NeedCarrier { - target: *target, - holders: holders.clone(), - }) - .collect(); - Ok(( - core, - ShadowOpenReportV1 { - replayed_batches, - discarded_tail_bytes, - recovery_effects, - }, - )) - } - - pub(crate) fn local_carrier_round(&self) -> RoundNumber { - self.model.local_carrier_round() - } - - pub(crate) fn can_create_carrier(&self) -> bool { - self.model.can_create_carrier() - } - - pub(crate) fn wal_counts(&self) -> (u64, u64) { - (self.wal.batch_count(), self.wal.record_count()) - } - - #[cfg(test)] - pub(crate) fn delivered( - &self, - authority: AuthorityIndex, - round: RoundNumber, - ) -> Option { - self.model.delivered(authority, round) - } - - /// Construct, authenticate, durably fix, and expose the next local - /// carrier. M3 intentionally uses empty ACKs and no consensus vertex. - pub(crate) fn create_local_carrier( - &mut self, - round: RoundNumber, - transactions_commitment: TransactionsCommitment, - creation_time_ns: TimestampNs, - ) -> Result<(ShadowOutboundEnvelopeV1, Vec), ShadowErrorV1> { - self.ensure_live()?; - let (own_prev, weak_parents) = self.model.local_parent_set()?; - let candidate = CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author: self.own_authority, - carrier_round: round, - own_prev, - weak_parents, - transactions_commitment, - data_acknowledgments: Vec::new(), - phase_batch: self.model.pending_phase_batch(), - consensus_vertex: None, - creation_time_ns, - }, - &self.committee, - )?; - let authenticated = self.context.authenticate_local_with_committee( - candidate, - &self.committee, - self.authorizer.authorizer(self.own_authority), - )?; - let envelope = ShadowOutboundEnvelopeV1 { - reference: authenticated.reference(), - canonical_carrier_wire: authenticated.candidate().canonical_wire_bytes()?, - authentication_sidecar: authenticated.authentication().canonical_wire_bytes(), - }; - let effects = self.apply_durable(ShadowInputV1::LocalOutbound(authenticated))?; - debug_assert!( - self.journal - .snapshot() - .outbound(envelope.reference()) - .is_some_and(|outbound| outbound.exposed()) - ); - Ok((envelope, effects)) - } - - /// Verify and durably apply an authenticated network envelope for this - /// exact receiver. - #[cfg(test)] - pub(crate) fn receive_authenticated_envelope( - &mut self, - canonical_carrier_wire: &[u8], - authentication_sidecar: &[u8], - provenance: IngressProvenanceV1, - ) -> Result, ShadowErrorV1> { - self.ensure_live()?; - let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; - validate_provenance(provenance, candidate.header().author(), &self.committee)?; - self.receive_decoded_authenticated(candidate, authentication_sidecar, provenance) - } - - /// Classify provenance from the authenticated transport peer and the - /// already-decoded carrier author, then verify and apply the envelope. - /// The candidate bytes are decoded exactly once in this method. - #[cfg(test)] - pub(crate) fn receive_authenticated_from_peer( - &mut self, - canonical_carrier_wire: &[u8], - authentication_sidecar: &[u8], - trusted_peer: AuthorityIndex, - ) -> Result, ShadowErrorV1> { - self.ensure_live()?; - if !self.committee.committee().known_authority(trusted_peer) { - return Err(ShadowErrorV1::UnknownAuthority(trusted_peer)); - } - let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; - let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); - self.receive_decoded_authenticated(candidate, authentication_sidecar, provenance) - } - - /// Verify an envelope when possible and otherwise durably retain its - /// canonical content as candidate-only input. This is the normal network - /// ingress API: a poisoned receiver tag must not discard the content that - /// embedded ECHO/READY evidence can later deliver. - pub(crate) fn receive_or_retain_from_peer( - &mut self, - canonical_carrier_wire: &[u8], - authentication_sidecar: &[u8], - trusted_peer: AuthorityIndex, - ) -> Result { - self.ensure_live()?; - if !self.committee.committee().known_authority(trusted_peer) { - return Err(ShadowErrorV1::UnknownAuthority(trusted_peer)); - } - let candidate = decode_candidate(canonical_carrier_wire, &self.committee, None)?; - let provenance = infer_ingress_provenance(trusted_peer, candidate.header().author()); - // Once a slot has a durably authenticated value, unsolicited replays - // and conflicts cannot change the shadow state. Reject before public - // signature/ML-DSA verification to keep this idempotence cheap. - if self.ignores_unsolicited_authenticated(&candidate) { - return Ok(ShadowIngressOutcomeV1::new( - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale, - Vec::new(), - )); - } - match self.authenticate_decoded(candidate.clone(), authentication_sidecar) { - Ok(authenticated) => { - let (effects, applied) = - self.apply_authenticated_capability(authenticated, provenance)?; - let disposition = if applied { - ShadowIngressDispositionV1::Authenticated - } else { - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale - }; - Ok(ShadowIngressOutcomeV1::new(disposition, effects)) - } - Err(ShadowErrorV1::Carrier(_)) | Err(ShadowErrorV1::NonCanonicalAuthentication) => { - let (effects, applied) = self.apply_candidate_retention(candidate)?; - let disposition = if applied { - ShadowIngressDispositionV1::CandidateRetained - } else { - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale - }; - Ok(ShadowIngressOutcomeV1::new(disposition, effects)) - } - Err(error) => Err(error), - } - } - - #[cfg(test)] - fn receive_decoded_authenticated( - &mut self, - candidate: CandidateCarrierV1, - authentication_sidecar: &[u8], - provenance: IngressProvenanceV1, - ) -> Result, ShadowErrorV1> { - let authenticated = self.authenticate_decoded(candidate, authentication_sidecar)?; - self.apply_authenticated_capability(authenticated, provenance) - .map(|(effects, _)| effects) - } - - fn authenticate_decoded( - &self, - candidate: CandidateCarrierV1, - authentication_sidecar: &[u8], - ) -> Result { - let authentication = decode_authentication(authentication_sidecar, &self.committee)?; - Ok(self.context.verify_authentication_with_committee( - candidate, - authentication, - self.own_authority, - &self.committee, - self.authorizer.inbound_mac_keys(), - )?) - } - - fn apply_authenticated_capability( - &mut self, - authenticated: AuthenticatedCarrierV1, - provenance: IngressProvenanceV1, - ) -> Result<(Vec, bool), ShadowErrorV1> { - let reference = authenticated.reference(); - let slot = carrier_slot(reference); - if round_is_stale(self.model.local_carrier_round(), reference.round) - || self.authenticated_slots.contains_key(&slot) - { - return Ok((Vec::new(), false)); - } - self.apply_durable(ShadowInputV1::AuthenticatedIngress { - authenticated, - provenance, - }) - .map(|effects| (effects, true)) - } - - fn ignores_unsolicited_authenticated(&self, candidate: &CandidateCarrierV1) -> bool { - let reference = candidate.reference(); - round_is_stale(self.model.local_carrier_round(), reference.round) - || self - .authenticated_slots - .contains_key(&carrier_slot(reference)) - } - - fn apply_candidate_retention( - &mut self, - candidate: CandidateCarrierV1, - ) -> Result<(Vec, bool), ShadowErrorV1> { - let reference = candidate.reference(); - let slot = carrier_slot(reference); - if round_is_stale(self.model.local_carrier_round(), reference.round) - || self.candidates.contains_key(&reference) - || self.authenticated_slots.contains_key(&slot) - || self.ordinarily_retained_slots.contains_key(&slot) - { - return Ok((Vec::new(), false)); - } - self.apply_durable(ShadowInputV1::CandidateRetention(candidate)) - .map(|effects| (effects, true)) - } - - /// Recover content only if it recomputes the exact requested reference. - pub(crate) fn recover_candidate_for( - &mut self, - expected_reference: BlockReference, - canonical_carrier_wire: &[u8], - ) -> Result, ShadowErrorV1> { - self.ensure_live()?; - let candidate = decode_candidate( - canonical_carrier_wire, - &self.committee, - Some(expected_reference), - )?; - self.apply_requested_recovery(candidate) - } - - pub(crate) fn retained_candidate_wire(&self, reference: BlockReference) -> Option> { - self.journal - .snapshot() - .retained_carrier(reference) - .map(<[u8]>::to_vec) - } - - /// Return every exposed local carrier in deterministic reference order. - /// The same full sidecar is returned for every peer. - pub(crate) fn retransmissions(&self) -> Vec { - self.journal - .snapshot() - .retransmissions() - .into_iter() - .map(|outbound| ShadowOutboundEnvelopeV1 { - reference: outbound.reference(), - canonical_carrier_wire: outbound.canonical_carrier_wire().to_vec(), - authentication_sidecar: outbound.authentication_sidecar().to_vec(), - }) - .collect() - } - - /// Metadata for every durably exposed local carrier, ordered by its exact - /// persisted reference. Startup uses this to prove that the shadow WAL and - /// recovered authoritative local chain overlap on the same application - /// payload and creation timestamp before accepting new observations. - pub(crate) fn local_outbound_metadata( - &self, - ) -> Result, ShadowErrorV1> { - self.journal - .snapshot() - .retransmissions() - .into_iter() - .map(|outbound| { - let reference = outbound.reference(); - let candidate = self - .candidates - .get(&reference) - .ok_or(ShadowErrorV1::MissingOutboundCandidate(reference))?; - Ok(( - candidate.header().carrier_round(), - candidate.header().transactions_commitment(), - candidate.header().creation_time_ns(), - )) - }) - .collect() - } - - pub(crate) fn delivered_identities( - &self, - ) -> Result, ShadowErrorV1> { - self.delivered - .iter() - .map(|reference| { - let candidate = self - .candidates - .get(reference) - .ok_or(ShadowErrorV1::MissingDeliveredCandidate(*reference))?; - Ok(ShadowDeliveryIdentityV1::new( - candidate.header().author(), - candidate.header().carrier_round(), - candidate.header().transactions_commitment(), - )) - }) - .collect() - } - - /// Compare protocol-independent delivery sets. Multiple transaction - /// commitments for one `(author, round)` slot make the comparison - /// ambiguous instead of being resolved by arrival or reference order. - #[cfg(test)] - pub(crate) fn compare_direct_deliveries( - &self, - direct: I, - ) -> Result - where - I: IntoIterator, - { - let direct: BTreeSet<_> = direct.into_iter().collect(); - let shadow: BTreeSet<_> = self.delivered_identities()?.into_iter().collect(); - let ambiguous = ambiguous_slots(&direct) - .into_iter() - .chain(ambiguous_slots(&shadow)) - .collect::>(); - if !ambiguous.is_empty() { - return Ok(ShadowDeliveryComparisonV1::Ambiguous { - slots: ambiguous.into_iter().collect(), - }); - } - if direct == shadow { - return Ok(ShadowDeliveryComparisonV1::Match); - } - Ok(ShadowDeliveryComparisonV1::Mismatch { - direct_only: direct.difference(&shadow).copied().collect(), - shadow_only: shadow.difference(&direct).copied().collect(), - }) - } - - pub(crate) fn shutdown(self) -> Result { - Ok(self.wal.shutdown()?) - } - - fn ensure_live(&self) -> Result<(), ShadowErrorV1> { - if self.poisoned || self.wal.is_poisoned() { - Err(ShadowErrorV1::Poisoned) - } else { - Ok(()) - } - } - - fn apply_requested_recovery( - &mut self, - candidate: CandidateCarrierV1, - ) -> Result, ShadowErrorV1> { - let reference = candidate.reference(); - if self.candidates.contains_key(&reference) { - return Ok(Vec::new()); - } - if !self.requested_recoveries.contains_key(&reference) { - return Err(ShadowErrorV1::UnrequestedRecovery(reference)); - } - let slot = carrier_slot(reference); - let limit = self - .committee - .committee() - .len() - .saturating_mul(2) - .saturating_add(2); - if self - .slot_candidates - .get(&slot) - .is_some_and(|candidates| candidates.len() >= limit) - { - return Err(ShadowErrorV1::SlotCandidateLimit { - author: reference.authority, - round: reference.round, - limit, - }); - } - self.apply_durable(ShadowInputV1::CandidateRecovery(candidate)) - } - - fn apply_durable(&mut self, input: ShadowInputV1) -> Result, ShadowErrorV1> { - let plan = self.model.plan_input(input.model_input())?; - let records = encode_batch(self.context, self.own_authority, &input, plan.trace())?; - let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; - self.wal.append_batch(&records)?; - let effects = match self.model.commit_plan(plan) { - Ok(effects) => effects, - Err(error) => { - self.poisoned = true; - return Err(ShadowErrorV1::PostDurabilityCommit(error)); - } - }; - if let Err(error) = self.journal.commit_validated_batch(journal_batch) { - self.poisoned = true; - return Err(ShadowErrorV1::PostDurabilityJournal(error)); - } - self.record_committed_input(&input, &effects); - Ok(effects) - } - - fn apply_replayed( - &mut self, - input: ShadowInputV1, - records: &[Vec], - batch_sequence: u64, - ) -> Result, ShadowErrorV1> { - self.validate_replay_policy(&input, batch_sequence)?; - let recorded_trace = decode_recorded_trace( - records, - self.context, - self.own_authority, - self.committee.committee().len(), - )?; - let plan = self.model.plan_input(input.model_input())?; - if plan.trace() != recorded_trace { - return Err(ShadowErrorV1::TraceMismatch { batch_sequence }); - } - let journal_batch = validate_journal_transition(&self.journal, &input, plan.trace())?; - let effects = self.model.commit_plan(plan)?; - if let Err(error) = self.journal.commit_validated_batch(journal_batch) { - self.poisoned = true; - return Err(ShadowErrorV1::PostDurabilityJournal(error)); - } - self.record_committed_input(&input, &effects); - Ok(effects) - } - - fn validate_replay_policy( - &self, - input: &ShadowInputV1, - batch_sequence: u64, - ) -> Result<(), ShadowErrorV1> { - let reference = input.candidate().reference(); - let slot = carrier_slot(reference); - let violation = match input { - ShadowInputV1::AuthenticatedIngress { .. } => { - if round_is_stale(self.model.local_carrier_round(), reference.round) { - Some("stale authenticated ingress") - } else if self.authenticated_slots.contains_key(&slot) { - Some("duplicate or conflicting authenticated slot") - } else { - None - } - } - ShadowInputV1::CandidateRetention(_) => { - if round_is_stale(self.model.local_carrier_round(), reference.round) { - Some("stale candidate retention") - } else if self.candidates.contains_key(&reference) - || self.authenticated_slots.contains_key(&slot) - || self.ordinarily_retained_slots.contains_key(&slot) - { - Some("duplicate or conflicting ordinary retention") - } else { - None - } - } - ShadowInputV1::CandidateRecovery(_) => { - let limit = self - .committee - .committee() - .len() - .saturating_mul(2) - .saturating_add(2); - if self.candidates.contains_key(&reference) { - Some("duplicate candidate recovery") - } else if !self.requested_recoveries.contains_key(&reference) { - Some("candidate recovery was not requested") - } else if self - .slot_candidates - .get(&slot) - .is_some_and(|candidates| candidates.len() >= limit) - { - Some("candidate recovery exceeds the per-slot limit") - } else { - None - } - } - ShadowInputV1::LocalOutbound(_) => None, - }; - if let Some(reason) = violation { - return Err(ShadowErrorV1::ReplayPolicyViolation { - batch_sequence, - reason, - }); - } - Ok(()) - } - - fn record_committed_input(&mut self, input: &ShadowInputV1, effects: &[ModelEffect]) { - let candidate = input.candidate().clone(); - let reference = candidate.reference(); - let slot = carrier_slot(reference); - self.candidates.insert(reference, candidate); - self.slot_candidates - .entry(slot) - .or_default() - .insert(reference); - match input { - ShadowInputV1::AuthenticatedIngress { .. } | ShadowInputV1::LocalOutbound(_) => { - self.authenticated_slots.entry(slot).or_insert(reference); - } - ShadowInputV1::CandidateRetention(_) => { - self.ordinarily_retained_slots - .entry(slot) - .or_insert(reference); - } - ShadowInputV1::CandidateRecovery(_) => {} - } - self.requested_recoveries.remove(&reference); - for effect in effects { - match effect { - ModelEffect::NeedCarrier { target, holders } => { - self.requested_recoveries.insert(*target, holders.clone()); - } - ModelEffect::Delivered(delivered) => { - self.delivered.insert(*delivered); - self.requested_recoveries.remove(delivered); - } - ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} - } - } - } - - fn decode_batch(&self, records: &[Vec]) -> Result { - if records.is_empty() { - return Err(ShadowErrorV1::InvalidBatch("empty transition batch")); - } - let decoded = records - .iter() - .map(|record| { - decode_raw_record(record, self.context, self.own_authority) - .map_err(ShadowErrorV1::from) - }) - .collect::, _>>()?; - match decoded[0].kind { - RECORD_AUTHENTICATED_INGRESS => { - ensure_trace_tail(&decoded[1..])?; - let mut payload = RawDecoder::new(&decoded[0].payload); - let provenance = decode_provenance(&mut payload)?; - let carrier_wire = payload.read_sized_bytes()?.to_vec(); - let sidecar_wire = payload.read_sized_bytes()?.to_vec(); - payload.finish()?; - let candidate = decode_candidate(&carrier_wire, &self.committee, None)?; - validate_provenance(provenance, candidate.header().author(), &self.committee)?; - let authentication = decode_authentication(&sidecar_wire, &self.committee)?; - let authenticated = self.context.verify_authentication_with_committee( - candidate, - authentication, - self.own_authority, - &self.committee, - self.authorizer.inbound_mac_keys(), - )?; - Ok(ShadowInputV1::AuthenticatedIngress { - authenticated, - provenance, - }) - } - RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY => { - ensure_trace_tail(&decoded[1..])?; - let candidate = decode_candidate(&decoded[0].payload, &self.committee, None)?; - if decoded[0].kind == RECORD_CANDIDATE_RETENTION { - Ok(ShadowInputV1::CandidateRetention(candidate)) - } else { - Ok(ShadowInputV1::CandidateRecovery(candidate)) - } - } - RECORD_LOCAL_OUTBOUND_CONTENT => { - if decoded.len() != 4 { - return Err(ShadowErrorV1::InvalidBatch( - "local transition must contain input, trace, sidecar, and exposure", - )); - } - let trace_end = decoded.len() - 2; - ensure_trace_tail(&decoded[1..trace_end])?; - if decoded[trace_end].kind != RECORD_LOCAL_OUTBOUND_SIDECAR - || decoded[trace_end + 1].kind != RECORD_LOCAL_OUTBOUND_EXPOSE - { - return Err(ShadowErrorV1::InvalidBatch( - "local sidecar and exposure must follow the trace", - )); - } - let candidate = decode_candidate(&decoded[0].payload, &self.committee, None)?; - let mut sidecar = RawDecoder::new(&decoded[trace_end].payload); - let sidecar_reference = sidecar.read_reference()?; - let authentication_wire = sidecar.read_sized_bytes()?.to_vec(); - sidecar.finish()?; - let mut expose = RawDecoder::new(&decoded[trace_end + 1].payload); - let expose_reference = expose.read_reference()?; - expose.finish()?; - if sidecar_reference != candidate.reference() - || expose_reference != candidate.reference() - { - return Err(ShadowErrorV1::InvalidBatch( - "local sidecar or exposure reference mismatch", - )); - } - let authentication = decode_authentication(&authentication_wire, &self.committee)?; - let authenticated = self.context.verify_local_authentication_with_committee( - candidate, - authentication, - &self.committee, - self.authorizer.authorizer(self.own_authority), - )?; - Ok(ShadowInputV1::LocalOutbound(authenticated)) - } - _ => Err(ShadowErrorV1::InvalidBatch( - "first record is not a model input", - )), - } - } -} - -fn validate_configuration( - committee: &RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: &ShadowAuthorizerV1, -) -> Result<(), ShadowErrorV1> { - if context.committee_id() != committee.committee_id() { - return Err(ShadowErrorV1::ContextMismatch); - } - if !committee.committee().known_authority(own_authority) { - return Err(ShadowErrorV1::UnknownAuthority(own_authority)); - } - if context.authentication_scheme() != authorizer.scheme() { - return Err(ShadowErrorV1::AuthorizerSchemeMismatch); - } - match authorizer { - ShadowAuthorizerV1::Ed25519(signer) => { - if committee.committee().get_public_key(own_authority) != Some(&signer.public_key()) { - return Err(ShadowErrorV1::AuthorizerKeyMismatch); - } - } - ShadowAuthorizerV1::MlDsa44(signer) => { - if committee - .committee() - .get_ml_dsa_44_public_key(own_authority) - != Some(&signer.public_key()) - { - return Err(ShadowErrorV1::AuthorizerKeyMismatch); - } - } - ShadowAuthorizerV1::MlDsa65(signer) => { - if committee - .committee() - .get_ml_dsa_65_public_key(own_authority) - != Some(&signer.public_key()) - { - return Err(ShadowErrorV1::AuthorizerKeyMismatch); - } - } - ShadowAuthorizerV1::MacVector(keys) => { - if keys.len() != committee.committee().len() { - return Err(ShadowErrorV1::InvalidAuthorizerKeyringLength { - expected: committee.committee().len(), - actual: keys.len(), - }); - } - } - } - Ok(()) -} - -fn validate_provenance( - provenance: IngressProvenanceV1, - candidate_author: AuthorityIndex, - committee: &RbcDagCommitteeContextV1, -) -> Result<(), ShadowErrorV1> { - if let IngressProvenanceV1::Relayed { peer } = provenance { - if !committee.committee().known_authority(peer) { - return Err(ShadowErrorV1::UnknownAuthority(peer)); - } - if peer == candidate_author { - return Err(ShadowErrorV1::NonCanonicalProvenance); - } - } - Ok(()) -} - -pub(crate) fn infer_ingress_provenance( - trusted_peer: AuthorityIndex, - candidate_author: AuthorityIndex, -) -> IngressProvenanceV1 { - if trusted_peer == candidate_author { - IngressProvenanceV1::DirectFromAuthor - } else { - IngressProvenanceV1::Relayed { peer: trusted_peer } - } -} - -fn decode_candidate( - wire: &[u8], - committee: &RbcDagCommitteeContextV1, - expected_reference: Option, -) -> Result { - let candidate = - CandidateCarrierV1::decode_wire_with_committee(wire, committee, expected_reference)?; - if candidate.canonical_wire_bytes()?.as_slice() != wire { - return Err(ShadowErrorV1::NonCanonicalCarrier); - } - Ok(candidate) -} - -fn decode_authentication( - wire: &[u8], - committee: &RbcDagCommitteeContextV1, -) -> Result { - let authentication = CarrierAuthenticationV1::decode_wire_with_committee(wire, committee)?; - if authentication.canonical_wire_bytes().as_slice() != wire { - return Err(ShadowErrorV1::NonCanonicalAuthentication); - } - Ok(authentication) -} - -fn validate_journal_transition( - journal: &WriteAheadJournalV1, - input: &ShadowInputV1, - trace: &[ModelTraceEvent], -) -> Result { - let mut events = Vec::new(); - let context = journal.snapshot().context(); - match input { - ShadowInputV1::AuthenticatedIngress { - authenticated, - provenance, - } => events.push(JournalEventV1::AuthenticatedIngress { - context, - sequence: journal.snapshot().next_ingress_sequence(), - authenticated: authenticated.clone(), - provenance: *provenance, - }), - ShadowInputV1::CandidateRetention(candidate) - | ShadowInputV1::CandidateRecovery(candidate) => { - events.push(JournalEventV1::RetainCandidateContent { - context, - candidate: candidate.clone(), - }); - } - ShadowInputV1::LocalOutbound(authenticated) => { - events.push(JournalEventV1::PersistOutboundContent { - context, - candidate: authenticated.candidate().clone(), - }); - } - } - - let local_reference = input.is_local().then(|| input.candidate().reference()); - for entry in trace { - let event = match entry { - ModelTraceEvent::AdmissionLocked(target) if Some(*target) == local_reference => { - // `FixOwnCarrier` is the journal's authority lock for a local - // value. `LockAdmission` intentionally accepts only network - // ingress; the exact model trace remains present in the WAL. - None - } - ModelTraceEvent::AdmissionLocked(target) => Some(JournalEventV1::LockAdmission { - context, - target: *target, - }), - ModelTraceEvent::LocalPhaseLocked(statement) => Some(match statement { - RbcPhaseStatementV1::Echo { target } => JournalEventV1::LockEcho { - context, - target: *target, - }, - RbcPhaseStatementV1::Ready { target } => JournalEventV1::LockReady { - context, - target: *target, - }, - }), - ModelTraceEvent::PhaseBatchEntryApplied { - outer, - index, - sender, - statement, - } => Some(JournalEventV1::ApplyPhaseStatement { - context, - outer: *outer, - index: *index, - sender: *sender, - statement: *statement, - }), - ModelTraceEvent::PhaseBatchCursorAdvanced { - outer, - index, - next_index, - } => { - if index.checked_add(1) != Some(*next_index) { - return Err(ShadowErrorV1::InvalidBatch( - "non-sequential phase cursor trace", - )); - } - Some(JournalEventV1::AdvancePhaseBatchCursor { - context, - outer: *outer, - index: *index, - }) - } - ModelTraceEvent::LocalCarrierFixed(reference) => Some(JournalEventV1::FixOwnCarrier { - context, - reference: *reference, - }), - ModelTraceEvent::DeliveryLocked(target) => Some(JournalEventV1::LockDelivery { - context, - target: *target, - }), - ModelTraceEvent::Effect(_) => None, - }; - if let Some(event) = event { - events.push(event); - } - } - - if let ShadowInputV1::LocalOutbound(authenticated) = input { - events.push(JournalEventV1::PersistOutboundSidecar { - context, - authenticated: authenticated.clone(), - }); - events.push(JournalEventV1::ExposeOutbound { - context, - reference: authenticated.reference(), - }); - } - journal.validate_batch(events).map_err(Into::into) -} - -fn encode_batch( - context: RbcDagContextV1, - own_authority: AuthorityIndex, - input: &ShadowInputV1, - trace: &[ModelTraceEvent], -) -> Result>, ShadowErrorV1> { - let mut records = Vec::with_capacity(4); - match input { - ShadowInputV1::AuthenticatedIngress { - authenticated, - provenance, - } => { - let mut payload = Vec::new(); - encode_provenance(&mut payload, *provenance); - push_sized_bytes( - &mut payload, - &authenticated.candidate().canonical_wire_bytes()?, - )?; - push_sized_bytes( - &mut payload, - &authenticated.authentication().canonical_wire_bytes(), - )?; - records.push(encode_raw_record( - context, - own_authority, - RECORD_AUTHENTICATED_INGRESS, - &payload, - )?); - } - ShadowInputV1::CandidateRetention(candidate) => { - records.push(encode_raw_record( - context, - own_authority, - RECORD_CANDIDATE_RETENTION, - &candidate.canonical_wire_bytes()?, - )?); - } - ShadowInputV1::CandidateRecovery(candidate) => { - records.push(encode_raw_record( - context, - own_authority, - RECORD_CANDIDATE_RECOVERY, - &candidate.canonical_wire_bytes()?, - )?); - } - ShadowInputV1::LocalOutbound(authenticated) => { - records.push(encode_raw_record( - context, - own_authority, - RECORD_LOCAL_OUTBOUND_CONTENT, - &authenticated.candidate().canonical_wire_bytes()?, - )?); - } - } - records.push(encode_raw_record( - context, - own_authority, - RECORD_MODEL_TRACE, - &encode_trace_batch(trace)?, - )?); - if let ShadowInputV1::LocalOutbound(authenticated) = input { - let mut sidecar = Vec::new(); - push_reference(&mut sidecar, authenticated.reference()); - push_sized_bytes( - &mut sidecar, - &authenticated.authentication().canonical_wire_bytes(), - )?; - records.push(encode_raw_record( - context, - own_authority, - RECORD_LOCAL_OUTBOUND_SIDECAR, - &sidecar, - )?); - let mut expose = Vec::new(); - push_reference(&mut expose, authenticated.reference()); - records.push(encode_raw_record( - context, - own_authority, - RECORD_LOCAL_OUTBOUND_EXPOSE, - &expose, - )?); - } - Ok(records) -} - -fn encode_raw_record( - context: RbcDagContextV1, - own_authority: AuthorityIndex, - kind: u8, - payload: &[u8], -) -> Result, ShadowCodecErrorV1> { - let payload_len = - u32::try_from(payload.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; - let total_len = RAW_RECORD_HEADER_SIZE - .checked_add(payload.len()) - .ok_or(ShadowCodecErrorV1::LengthOverflow)?; - if total_len > MAX_SHADOW_WAL_RECORD_SIZE_V1 { - return Err(ShadowCodecErrorV1::InvalidRecordLength(total_len)); - } - let mut bytes = Vec::with_capacity(total_len); - bytes.extend_from_slice(RAW_RECORD_MAGIC); - bytes.push(RAW_RECORD_VERSION_V1); - bytes.push(kind); - bytes.extend_from_slice(&0u16.to_be_bytes()); - bytes.extend_from_slice(context.protocol_instance().as_bytes()); - bytes.extend_from_slice(context.committee_id().as_bytes()); - bytes.extend_from_slice(&own_authority.to_be_bytes()); - bytes.push(authentication_scheme_code(context.authentication_scheme())); - bytes.push(0); - bytes.extend_from_slice(&payload_len.to_be_bytes()); - debug_assert_eq!(bytes.len(), RAW_RECORD_HEADER_SIZE); - bytes.extend_from_slice(payload); - Ok(bytes) -} - -fn decode_raw_record( - bytes: &[u8], - context: RbcDagContextV1, - own_authority: AuthorityIndex, -) -> Result { - if bytes.len() > MAX_SHADOW_WAL_RECORD_SIZE_V1 { - return Err(ShadowCodecErrorV1::InvalidRecordLength(bytes.len())); - } - let mut decoder = RawDecoder::new(bytes); - if decoder.take(4)? != RAW_RECORD_MAGIC { - return Err(ShadowCodecErrorV1::InvalidMagic); - } - let version = decoder.read_u8()?; - if version != RAW_RECORD_VERSION_V1 { - return Err(ShadowCodecErrorV1::UnsupportedVersion(version)); - } - let kind = decoder.read_u8()?; - let flags = decoder.read_u16()?; - if flags != 0 { - return Err(ShadowCodecErrorV1::InvalidFlags(flags)); - } - if decoder.take(32)? != context.protocol_instance().as_bytes() - || decoder.take(32)? != context.committee_id().as_bytes() - { - return Err(ShadowCodecErrorV1::ContextMismatch); - } - let actual_authority = decoder.read_u16()?; - if actual_authority != own_authority { - return Err(ShadowCodecErrorV1::AuthorityMismatch { - expected: own_authority, - actual: actual_authority, - }); - } - if decoder.read_u8()? != authentication_scheme_code(context.authentication_scheme()) { - return Err(ShadowCodecErrorV1::AuthenticationSchemeMismatch); - } - if decoder.read_u8()? != 0 { - return Err(ShadowCodecErrorV1::InvalidFlags(1)); - } - let payload_len = decoder.read_u32()? as usize; - if payload_len != decoder.remaining() { - return Err(ShadowCodecErrorV1::InvalidRecordLength(bytes.len())); - } - let payload = decoder.take(payload_len)?.to_vec(); - decoder.finish()?; - match kind { - RECORD_AUTHENTICATED_INGRESS - | RECORD_CANDIDATE_RETENTION - | RECORD_CANDIDATE_RECOVERY - | RECORD_LOCAL_OUTBOUND_CONTENT - | RECORD_MODEL_TRACE - | RECORD_LOCAL_OUTBOUND_SIDECAR - | RECORD_LOCAL_OUTBOUND_EXPOSE => {} - other => return Err(ShadowCodecErrorV1::UnknownRecordKind(other)), - } - Ok(DecodedRawRecord { kind, payload }) -} - -fn decode_recorded_trace( - records: &[Vec], - context: RbcDagContextV1, - own_authority: AuthorityIndex, - committee_size: usize, -) -> Result, ShadowErrorV1> { - let decoded = records - .iter() - .map(|record| decode_raw_record(record, context, own_authority)) - .collect::, _>>()?; - let range = match decoded.first().map(|record| record.kind) { - Some(RECORD_LOCAL_OUTBOUND_CONTENT) => 1..decoded.len().saturating_sub(2), - Some( - RECORD_AUTHENTICATED_INGRESS | RECORD_CANDIDATE_RETENTION | RECORD_CANDIDATE_RECOVERY, - ) => 1..decoded.len(), - _ => return Err(ShadowErrorV1::InvalidBatch("missing model input")), - }; - let trace_records = &decoded[range]; - ensure_trace_tail(trace_records)?; - decode_trace_batch(&trace_records[0].payload, committee_size).map_err(ShadowErrorV1::from) -} - -fn ensure_trace_tail(records: &[DecodedRawRecord]) -> Result<(), ShadowErrorV1> { - if matches!(records, [record] if record.kind == RECORD_MODEL_TRACE) { - Ok(()) - } else { - Err(ShadowErrorV1::InvalidBatch( - "each model input must have exactly one ordered trace record", - )) - } -} - -fn encode_trace_batch(trace: &[ModelTraceEvent]) -> Result, ShadowCodecErrorV1> { - let count = u32::try_from(trace.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; - let mut bytes = Vec::new(); - bytes.extend_from_slice(&count.to_be_bytes()); - for entry in trace { - push_sized_bytes(&mut bytes, &encode_trace(entry)?)?; - } - Ok(bytes) -} - -fn decode_trace_batch( - bytes: &[u8], - committee_size: usize, -) -> Result, ShadowCodecErrorV1> { - let mut decoder = RawDecoder::new(bytes); - let count = decoder.read_u32()? as usize; - if count > decoder.remaining() / 5 { - return Err(ShadowCodecErrorV1::InvalidRecordLength(bytes.len())); - } - let mut trace = Vec::with_capacity(count); - for _ in 0..count { - trace.push(decode_trace(decoder.read_sized_bytes()?, committee_size)?); - } - decoder.finish()?; - Ok(trace) -} - -fn encode_trace(trace: &ModelTraceEvent) -> Result, ShadowCodecErrorV1> { - let mut bytes = Vec::new(); - match trace { - ModelTraceEvent::AdmissionLocked(reference) => { - bytes.push(TRACE_ADMISSION_LOCKED); - push_reference(&mut bytes, *reference); - } - ModelTraceEvent::LocalPhaseLocked(statement) => { - bytes.push(TRACE_LOCAL_PHASE_LOCKED); - push_phase(&mut bytes, *statement); - } - ModelTraceEvent::PhaseBatchEntryApplied { - outer, - index, - sender, - statement, - } => { - bytes.push(TRACE_PHASE_ENTRY_APPLIED); - push_reference(&mut bytes, *outer); - push_usize_as_u32(&mut bytes, *index)?; - bytes.extend_from_slice(&sender.to_be_bytes()); - push_phase(&mut bytes, *statement); - } - ModelTraceEvent::PhaseBatchCursorAdvanced { - outer, - index, - next_index, - } => { - bytes.push(TRACE_PHASE_CURSOR_ADVANCED); - push_reference(&mut bytes, *outer); - push_usize_as_u32(&mut bytes, *index)?; - push_usize_as_u32(&mut bytes, *next_index)?; - } - ModelTraceEvent::LocalCarrierFixed(reference) => { - bytes.push(TRACE_LOCAL_CARRIER_FIXED); - push_reference(&mut bytes, *reference); - } - ModelTraceEvent::DeliveryLocked(reference) => { - bytes.push(TRACE_DELIVERY_LOCKED); - push_reference(&mut bytes, *reference); - } - ModelTraceEvent::Effect(effect) => { - bytes.push(TRACE_EFFECT); - push_effect(&mut bytes, effect)?; - } - } - Ok(bytes) -} - -fn decode_trace( - bytes: &[u8], - committee_size: usize, -) -> Result { - let mut decoder = RawDecoder::new(bytes); - let trace = match decoder.read_u8()? { - TRACE_ADMISSION_LOCKED => ModelTraceEvent::AdmissionLocked(decoder.read_reference()?), - TRACE_LOCAL_PHASE_LOCKED => ModelTraceEvent::LocalPhaseLocked(decoder.read_phase()?), - TRACE_PHASE_ENTRY_APPLIED => ModelTraceEvent::PhaseBatchEntryApplied { - outer: decoder.read_reference()?, - index: decoder.read_u32()? as usize, - sender: decoder.read_u16()?, - statement: decoder.read_phase()?, - }, - TRACE_PHASE_CURSOR_ADVANCED => ModelTraceEvent::PhaseBatchCursorAdvanced { - outer: decoder.read_reference()?, - index: decoder.read_u32()? as usize, - next_index: decoder.read_u32()? as usize, - }, - TRACE_LOCAL_CARRIER_FIXED => ModelTraceEvent::LocalCarrierFixed(decoder.read_reference()?), - TRACE_DELIVERY_LOCKED => ModelTraceEvent::DeliveryLocked(decoder.read_reference()?), - TRACE_EFFECT => ModelTraceEvent::Effect(decoder.read_effect(committee_size)?), - other => return Err(ShadowCodecErrorV1::InvalidTrace(other)), - }; - decoder.finish()?; - Ok(trace) -} - -fn push_effect(bytes: &mut Vec, effect: &ModelEffect) -> Result<(), ShadowCodecErrorV1> { - match effect { - ModelEffect::NeedCarrier { target, holders } => { - bytes.push(EFFECT_NEED_CARRIER); - push_reference(bytes, *target); - let count = - u16::try_from(holders.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; - bytes.extend_from_slice(&count.to_be_bytes()); - for holder in holders { - bytes.extend_from_slice(&holder.to_be_bytes()); - } - } - ModelEffect::Delivered(reference) => { - bytes.push(EFFECT_DELIVERED); - push_reference(bytes, *reference); - } - ModelEffect::PrefixAdvanced { authority, tip } => { - bytes.push(EFFECT_PREFIX_ADVANCED); - bytes.extend_from_slice(&authority.to_be_bytes()); - push_reference(bytes, *tip); - } - ModelEffect::CarrierRoundAdvanced(round) => { - bytes.push(EFFECT_CARRIER_ROUND_ADVANCED); - bytes.extend_from_slice(&round.to_be_bytes()); - } - } - Ok(()) -} - -fn push_phase(bytes: &mut Vec, statement: RbcPhaseStatementV1) { - match statement { - RbcPhaseStatementV1::Echo { target } => { - bytes.push(PHASE_ECHO); - push_reference(bytes, target); - } - RbcPhaseStatementV1::Ready { target } => { - bytes.push(PHASE_READY); - push_reference(bytes, target); - } - } -} - -fn encode_provenance(bytes: &mut Vec, provenance: IngressProvenanceV1) { - match provenance { - IngressProvenanceV1::DirectFromAuthor => bytes.push(PROVENANCE_DIRECT), - IngressProvenanceV1::Relayed { peer } => { - bytes.push(PROVENANCE_RELAYED); - bytes.extend_from_slice(&peer.to_be_bytes()); - } - } -} - -fn decode_provenance( - decoder: &mut RawDecoder<'_>, -) -> Result { - match decoder.read_u8()? { - PROVENANCE_DIRECT => Ok(IngressProvenanceV1::DirectFromAuthor), - PROVENANCE_RELAYED => Ok(IngressProvenanceV1::Relayed { - peer: decoder.read_u16()?, - }), - other => Err(ShadowCodecErrorV1::InvalidProvenance(other)), - } -} - -fn push_reference(bytes: &mut Vec, reference: BlockReference) { - bytes.extend_from_slice(&reference.authority.to_be_bytes()); - bytes.extend_from_slice(&reference.round.to_be_bytes()); - bytes.extend_from_slice(reference.digest.as_ref()); -} - -fn push_sized_bytes(target: &mut Vec, bytes: &[u8]) -> Result<(), ShadowCodecErrorV1> { - let len = u32::try_from(bytes.len()).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; - target.extend_from_slice(&len.to_be_bytes()); - target.extend_from_slice(bytes); - Ok(()) -} - -fn push_usize_as_u32(target: &mut Vec, value: usize) -> Result<(), ShadowCodecErrorV1> { - let value = u32::try_from(value).map_err(|_| ShadowCodecErrorV1::LengthOverflow)?; - target.extend_from_slice(&value.to_be_bytes()); - Ok(()) -} - -fn authentication_scheme_code(scheme: BlockAuthenticationScheme) -> u8 { - match scheme { - BlockAuthenticationScheme::Ed25519 => 0, - BlockAuthenticationScheme::MlDsa44 => 1, - BlockAuthenticationScheme::MlDsa65 => 2, - BlockAuthenticationScheme::MacVector => 3, - } -} - -fn carrier_slot(reference: BlockReference) -> (AuthorityIndex, RoundNumber) { - (reference.authority, reference.round) -} - -fn round_is_stale(current_round: RoundNumber, candidate_round: RoundNumber) -> bool { - candidate_round - < current_round.saturating_sub(SHADOW_BENCHMARK_UNSOLICITED_RETENTION_WINDOW_ROUNDS_V1) -} - -#[cfg(test)] -fn ambiguous_slots( - identities: &BTreeSet, -) -> BTreeSet { - let mut by_slot: BTreeMap> = - BTreeMap::new(); - for identity in identities { - by_slot - .entry(ShadowDeliverySlotV1 { - author: identity.author, - round: identity.round, - }) - .or_default() - .insert(identity.transactions_commitment); - } - by_slot - .into_iter() - .filter_map(|(slot, commitments)| (commitments.len() > 1).then_some(slot)) - .collect() -} - -struct RawDecoder<'a> { - bytes: &'a [u8], - position: usize, -} - -impl<'a> RawDecoder<'a> { - fn new(bytes: &'a [u8]) -> Self { - Self { bytes, position: 0 } - } - - fn remaining(&self) -> usize { - self.bytes.len().saturating_sub(self.position) - } - - fn take(&mut self, length: usize) -> Result<&'a [u8], ShadowCodecErrorV1> { - let end = self - .position - .checked_add(length) - .ok_or(ShadowCodecErrorV1::LengthOverflow)?; - let bytes = self - .bytes - .get(self.position..end) - .ok_or(ShadowCodecErrorV1::UnexpectedEnd)?; - self.position = end; - Ok(bytes) - } - - fn read_u8(&mut self) -> Result { - Ok(self.take(1)?[0]) - } - - fn read_u16(&mut self) -> Result { - Ok(u16::from_be_bytes( - self.take(2)?.try_into().expect("fixed u16 range"), - )) - } - - fn read_u32(&mut self) -> Result { - Ok(u32::from_be_bytes( - self.take(4)?.try_into().expect("fixed u32 range"), - )) - } - - fn read_reference(&mut self) -> Result { - let authority = self.read_u16()?; - let round = self.read_u32()?; - let mut digest = [0; 32]; - digest.copy_from_slice(self.take(32)?); - Ok(BlockReference { - authority, - round, - digest: BlockDigest::from(digest), - }) - } - - fn read_phase(&mut self) -> Result { - let phase = self.read_u8()?; - let target = self.read_reference()?; - match phase { - PHASE_ECHO => Ok(RbcPhaseStatementV1::Echo { target }), - PHASE_READY => Ok(RbcPhaseStatementV1::Ready { target }), - other => Err(ShadowCodecErrorV1::InvalidPhase(other)), - } - } - - fn read_effect(&mut self, committee_size: usize) -> Result { - match self.read_u8()? { - EFFECT_NEED_CARRIER => { - let target = self.read_reference()?; - let count = self.read_u16()? as usize; - if count > committee_size || count > MAX_COMMITTEE_SIZE as usize { - return Err(ShadowCodecErrorV1::NonCanonicalHolders); - } - let mut holders = Vec::with_capacity(count); - let mut previous = None; - for _ in 0..count { - let holder = self.read_u16()?; - if holder as usize >= committee_size - || previous.is_some_and(|previous| previous >= holder) - { - return Err(ShadowCodecErrorV1::NonCanonicalHolders); - } - previous = Some(holder); - holders.push(holder); - } - Ok(ModelEffect::NeedCarrier { target, holders }) - } - EFFECT_DELIVERED => Ok(ModelEffect::Delivered(self.read_reference()?)), - EFFECT_PREFIX_ADVANCED => Ok(ModelEffect::PrefixAdvanced { - authority: self.read_u16()?, - tip: self.read_reference()?, - }), - EFFECT_CARRIER_ROUND_ADVANCED => { - Ok(ModelEffect::CarrierRoundAdvanced(self.read_u32()?)) - } - other => Err(ShadowCodecErrorV1::InvalidEffect(other)), - } - } - - fn read_sized_bytes(&mut self) -> Result<&'a [u8], ShadowCodecErrorV1> { - let length = self.read_u32()? as usize; - self.take(length) - } - - fn finish(self) -> Result<(), ShadowCodecErrorV1> { - if self.position == self.bytes.len() { - Ok(()) - } else { - Err(ShadowCodecErrorV1::TrailingBytes( - self.bytes.len() - self.position, - )) - } - } -} - -const _: () = assert!(RAW_RECORD_HEADER_SIZE == 80); - -#[cfg(test)] -mod tests { - use super::*; - use std::{fs::OpenOptions, io::Write, sync::Arc}; - - use tempfile::TempDir; - - use crate::{ - committee::Committee, - crypto::{ - MAC_TAG_SIZE, dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, - mac_keyrings_for_test, - }, - starfish_rbc_dag::{RbcDagProtocolInstanceId, carrier_genesis_reference}, - }; - - const N: usize = 4; - - struct TestNetwork { - committee: RbcDagCommitteeContextV1, - context: RbcDagContextV1, - keyrings: Vec>, - directories: Vec, - nodes: Vec, - } - - impl TestNetwork { - fn new() -> Self { - let committee = Committee::new_test(vec![1; N]); - let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xE3; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ); - let keyrings = mac_keyrings_for_test(N); - let directories = (0..N) - .map(|_| tempfile::tempdir().unwrap()) - .collect::>(); - let nodes = (0..N) - .map(|authority| { - StarfishRbcDagShadowV1::open( - directories[authority].path().join("shadow.wal"), - committee.clone(), - authority as AuthorityIndex, - context, - ShadowAuthorizerV1::MacVector(keyrings[authority].clone()), - ) - .unwrap() - .0 - }) - .collect(); - Self { - committee, - context, - keyrings, - directories, - nodes, - } - } - - fn path(&self, authority: usize) -> std::path::PathBuf { - self.directories[authority].path().join("shadow.wal") - } - - fn run_three_rounds_with_one_poisoned_recipient(&mut self) { - for round in 1..=3 { - let envelopes = self - .nodes - .iter_mut() - .enumerate() - .map(|(authority, node)| { - let commitment = TransactionsCommitment::from_bytes( - [(round * 16 + authority as u32) as u8; 32], - ); - node.create_local_carrier(round, commitment, u64::from(round) * 1_000) - .unwrap() - .0 - }) - .collect::>(); - - for (sender, envelope) in envelopes.iter().enumerate() { - for receiver in 0..N { - if receiver == sender { - continue; - } - if round == 1 && sender == 0 && receiver == 2 { - let mut poisoned = envelope.authentication_sidecar().to_vec(); - poisoned[3 + receiver * MAC_TAG_SIZE] ^= 1; - let outcome = self.nodes[receiver] - .receive_or_retain_from_peer( - envelope.canonical_carrier_wire(), - &poisoned, - sender as AuthorityIndex, - ) - .unwrap(); - assert_eq!( - outcome.disposition(), - ShadowIngressDispositionV1::CandidateRetained - ); - assert!(outcome.effects().is_empty()); - } else { - self.nodes[receiver] - .receive_authenticated_from_peer( - envelope.canonical_carrier_wire(), - envelope.authentication_sidecar(), - sender as AuthorityIndex, - ) - .unwrap(); - } - } - } - - for node in &self.nodes { - assert_eq!(node.local_carrier_round(), round + 1); - } - } - } - } - - #[test] - fn three_round_mac_shadow_delivers_round_one_after_poisoned_tag_is_only_staged() { - let mut network = TestNetwork::new(); - network.run_three_rounds_with_one_poisoned_recipient(); - - for node in &network.nodes { - for author in 0..N { - assert!(node.delivered(author as AuthorityIndex, 1).is_some()); - } - } - assert!(network.nodes[2].delivered(0, 1).is_some()); - assert_eq!( - infer_ingress_provenance(1, 1), - IngressProvenanceV1::DirectFromAuthor - ); - assert_eq!( - infer_ingress_provenance(2, 1), - IngressProvenanceV1::Relayed { peer: 2 } - ); - } - - #[test] - fn authenticated_replays_and_slot_conflicts_do_not_grow_durable_state() { - let mut network = TestNetwork::new(); - let first = round_one_candidate(1, &network.committee, 0x71); - let first_authentication = network - .context - .authenticate_with_committee( - &first, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - let wire = first.canonical_wire_bytes().unwrap(); - let sidecar = first_authentication.canonical_wire_bytes(); - let accepted = network.nodes[0] - .receive_or_retain_from_peer(&wire, &sidecar, 1) - .unwrap(); - assert_eq!( - accepted.disposition(), - ShadowIngressDispositionV1::Authenticated - ); - let counts = network.nodes[0].wal_counts(); - - let replay = network.nodes[0] - .receive_or_retain_from_peer(&wire, &sidecar, 1) - .unwrap(); - assert_eq!( - replay.disposition(), - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale - ); - assert_eq!(network.nodes[0].wal_counts(), counts); - - let conflicting = round_one_candidate(1, &network.committee, 0x72); - let conflicting_authentication = network - .context - .authenticate_with_committee( - &conflicting, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - let conflict = network.nodes[0] - .receive_or_retain_from_peer( - &conflicting.canonical_wire_bytes().unwrap(), - &conflicting_authentication.canonical_wire_bytes(), - 1, - ) - .unwrap(); - assert_eq!( - conflict.disposition(), - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale - ); - assert_eq!(network.nodes[0].wal_counts(), counts); - - // The classified runtime path cheaply ignores an occupied slot, but - // the explicitly strict API must retain its advertised verification - // contract even when the candidate cannot affect state. - let mut invalid_conflict_sidecar = conflicting_authentication.canonical_wire_bytes(); - invalid_conflict_sidecar[3] ^= 1; - assert!(matches!( - network.nodes[0].receive_authenticated_from_peer( - &conflicting.canonical_wire_bytes().unwrap(), - &invalid_conflict_sidecar, - 1, - ), - Err(ShadowErrorV1::Carrier(_)) - )); - assert_eq!(network.nodes[0].wal_counts(), counts); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let (mut restarted, _) = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap(); - assert_eq!(restarted.wal_counts(), counts); - let replay_after_restart = restarted - .receive_or_retain_from_peer(&wire, &sidecar, 1) - .unwrap(); - assert_eq!( - replay_after_restart.disposition(), - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale - ); - assert_eq!(restarted.wal_counts(), counts); - } - - #[test] - fn unsolicited_stale_window_has_a_fixed_round_bound() { - assert!(!round_is_stale(64, 0)); - assert!(!round_is_stale(65, 1)); - assert!(round_is_stale(66, 1)); - assert!(!round_is_stale(66, 2)); - } - - #[test] - fn caller_supplied_relay_provenance_must_be_canonical_for_the_author() { - let mut network = TestNetwork::new(); - let candidate = round_one_candidate(1, &network.committee, 0x80); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - assert!(matches!( - network.nodes[0].receive_authenticated_envelope( - &candidate.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - IngressProvenanceV1::Relayed { peer: 1 }, - ), - Err(ShadowErrorV1::NonCanonicalProvenance) - )); - assert_eq!(network.nodes[0].wal_counts(), (0, 0)); - } - - #[test] - fn replay_rejects_a_duplicate_authenticated_slot_even_with_an_exact_trace() { - let mut network = TestNetwork::new(); - let candidate = round_one_candidate(1, &network.committee, 0x81); - let authentication = network - .context - .authenticate_with_committee( - &candidate, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &candidate.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - 1, - ) - .unwrap(); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let namespace = ShadowWalNamespaceV1::new(network.context, 0); - let (mut wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - let duplicate = recovery.batches()[0].records().to_vec(); - wal.append_batch(&duplicate).unwrap(); - wal.shutdown().unwrap(); - - let result = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ); - assert!(matches!( - result, - Err(ShadowErrorV1::ReplayPolicyViolation { - batch_sequence: 1, - reason: "duplicate or conflicting authenticated slot", - }) - )); - } - - #[test] - fn replay_rejects_candidate_recovery_that_was_never_requested() { - let mut network = TestNetwork::new(); - let target = round_one_candidate(3, &network.committee, 0x82); - let outer = round_two_phase_carrier( - 1, - RbcPhaseStatementV1::Echo { - target: target.reference(), - }, - &network.committee, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: 1, - keys: &network.keyrings[1], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - 1, - ) - .unwrap(); - assert!( - !network.nodes[0] - .requested_recoveries - .contains_key(&target.reference()) - ); - - // The reducer accepts content after any phase evidence allocated the - // candidate. The shadow's durable grammar is deliberately stricter: - // network recovery is legal only after a surfaced NeedCarrier. - let forged_input = ShadowInputV1::CandidateRecovery(target); - let plan = network.nodes[0] - .model - .plan_input(forged_input.model_input()) - .unwrap(); - let forged_batch = encode_batch(network.context, 0, &forged_input, plan.trace()).unwrap(); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - node.shutdown().unwrap(); - let namespace = ShadowWalNamespaceV1::new(network.context, 0); - let (mut wal, _) = ShadowWalV1::open(&path, namespace).unwrap(); - wal.append_batch(&forged_batch).unwrap(); - wal.shutdown().unwrap(); - - let result = StarfishRbcDagShadowV1::open( - path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ); - assert!(matches!( - result, - Err(ShadowErrorV1::ReplayPolicyViolation { - batch_sequence: 1, - reason: "candidate recovery was not requested", - }) - )); - } - - #[test] - fn wal_restart_past_round_one_discards_torn_tail_and_retransmits_exact_bytes() { - let mut network = TestNetwork::new(); - network.run_three_rounds_with_one_poisoned_recipient(); - - let node = network.nodes.swap_remove(0); - let path = network.path(0); - let before = node.retransmissions(); - assert_eq!(before.len(), 3); - let retained = node.retained_candidate_wire(before[0].reference()).unwrap(); - assert_eq!(retained, before[0].canonical_carrier_wire()); - node.shutdown().unwrap(); - - let namespace = ShadowWalNamespaceV1::new(network.context, 0); - let (wal, recovery) = ShadowWalV1::open(&path, namespace).unwrap(); - let first = recovery.batches().first().unwrap().records(); - let kinds = first - .iter() - .map(|record| decode_raw_record(record, network.context, 0).unwrap().kind) - .collect::>(); - assert_eq!(kinds[0], RECORD_LOCAL_OUTBOUND_CONTENT); - assert_eq!(kinds[kinds.len() - 2], RECORD_LOCAL_OUTBOUND_SIDECAR); - assert_eq!(kinds[kinds.len() - 1], RECORD_LOCAL_OUTBOUND_EXPOSE); - assert!( - kinds[1..kinds.len() - 2] - .iter() - .all(|kind| *kind == RECORD_MODEL_TRACE) - ); - wal.shutdown().unwrap(); - - let torn = b"uncommitted-tail"; - let mut file = OpenOptions::new().append(true).open(&path).unwrap(); - file.write_all(torn).unwrap(); - file.sync_all().unwrap(); - drop(file); - - let (restarted, report) = StarfishRbcDagShadowV1::open( - &path, - network.committee.clone(), - 0, - network.context, - ShadowAuthorizerV1::MacVector(network.keyrings[0].clone()), - ) - .unwrap(); - assert_eq!(report.discarded_tail_bytes(), torn.len() as u64); - assert!(report.replayed_batches() > 3); - assert_eq!(restarted.local_carrier_round(), 4); - assert_eq!(restarted.retransmissions(), before); - assert!(report.recovery_effects().is_empty()); - for author in 0..N { - assert!(restarted.delivered(author as AuthorityIndex, 1).is_some()); - } - } - - #[test] - fn every_authentication_scheme_reopens_with_the_exact_persisted_sidecar() { - let committee = Committee::new_test(vec![1; N]); - let committee = RbcDagCommitteeContextV1::new(committee).unwrap(); - let keyrings = mac_keyrings_for_test(N); - let directory = tempfile::tempdir().unwrap(); - - for (index, scheme) in [ - BlockAuthenticationScheme::Ed25519, - BlockAuthenticationScheme::MlDsa44, - BlockAuthenticationScheme::MlDsa65, - BlockAuthenticationScheme::MacVector, - ] - .into_iter() - .enumerate() - { - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xA0 + index as u8; 32]).unwrap(), - &committee, - scheme, - ); - let authorizer = match scheme { - BlockAuthenticationScheme::Ed25519 => ShadowAuthorizerV1::Ed25519(dummy_signer()), - BlockAuthenticationScheme::MlDsa44 => { - ShadowAuthorizerV1::MlDsa44(dummy_ml_dsa_44_signer()) - } - BlockAuthenticationScheme::MlDsa65 => { - ShadowAuthorizerV1::MlDsa65(dummy_ml_dsa_65_signer()) - } - BlockAuthenticationScheme::MacVector => { - ShadowAuthorizerV1::MacVector(keyrings[0].clone()) - } - }; - let path = directory.path().join(format!("scheme-{index}.wal")); - let (mut core, _) = StarfishRbcDagShadowV1::open( - &path, - committee.clone(), - 0, - context, - authorizer.clone(), - ) - .unwrap(); - let envelope = core - .create_local_carrier( - 1, - TransactionsCommitment::from_bytes([0xE0 + index as u8; 32]), - 10, - ) - .unwrap() - .0; - core.shutdown().unwrap(); - - let (restarted, report) = - StarfishRbcDagShadowV1::open(&path, committee.clone(), 0, context, authorizer) - .unwrap(); - assert!(report.replayed_batches() > 0); - assert_eq!(restarted.retransmissions(), vec![envelope]); - restarted.shutdown().unwrap(); - } - } - - #[test] - fn reopening_rejects_a_wrong_local_signer_before_replaying_the_wal() { - let committee = Committee::new_test(vec![1; N]); - let committee = RbcDagCommitteeContextV1::new(committee).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xAF; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::Ed25519, - ); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("wrong-reopen-signer.wal"); - let (mut core, _) = StarfishRbcDagShadowV1::open( - &path, - committee.clone(), - 0, - context, - ShadowAuthorizerV1::Ed25519(dummy_signer()), - ) - .unwrap(); - core.create_local_carrier(1, TransactionsCommitment::from_bytes([0xEF; 32]), 10) - .unwrap(); - core.shutdown().unwrap(); - - let wrong_signer = Signer::new_for_test(1).pop().unwrap(); - let result = StarfishRbcDagShadowV1::open( - path, - committee, - 0, - context, - ShadowAuthorizerV1::Ed25519(wrong_signer), - ); - assert!(matches!(result, Err(ShadowErrorV1::AuthorizerKeyMismatch))); - } - - #[test] - fn replay_rejects_a_validly_framed_corrupt_trace() { - let committee = Committee::new_test(vec![1; N]); - let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xF4; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ); - let keyrings = mac_keyrings_for_test(N); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("corrupt-trace.wal"); - let candidate = round_one_candidate(1, &committee, 0x91); - let input = encode_raw_record( - context, - 0, - RECORD_CANDIDATE_RETENTION, - &candidate.canonical_wire_bytes().unwrap(), - ) - .unwrap(); - let fabricated_trace = encode_raw_record( - context, - 0, - RECORD_MODEL_TRACE, - &encode_trace_batch(&[ModelTraceEvent::Effect(ModelEffect::CarrierRoundAdvanced( - 99, - ))]) - .unwrap(), - ) - .unwrap(); - let (mut wal, _) = ShadowWalV1::open(&path, ShadowWalNamespaceV1::new(context, 0)).unwrap(); - wal.append_batch(&[input, fabricated_trace]).unwrap(); - wal.shutdown().unwrap(); - - let result = StarfishRbcDagShadowV1::open( - &path, - committee, - 0, - context, - ShadowAuthorizerV1::MacVector(keyrings[0].clone()), - ); - assert!(matches!( - result, - Err(ShadowErrorV1::TraceMismatch { batch_sequence: 0 }) - )); - } - - #[test] - fn direct_shadow_comparison_reports_match_mismatch_and_ambiguity_without_references() { - let mut network = TestNetwork::new(); - network.run_three_rounds_with_one_poisoned_recipient(); - let node = &network.nodes[0]; - let direct = node.delivered_identities().unwrap(); - assert_eq!( - node.compare_direct_deliveries(direct.clone()).unwrap(), - ShadowDeliveryComparisonV1::Match - ); - - let mut mismatch = direct.clone(); - mismatch[0].transactions_commitment = TransactionsCommitment::from_bytes([0xAB; 32]); - assert!(matches!( - node.compare_direct_deliveries(mismatch).unwrap(), - ShadowDeliveryComparisonV1::Mismatch { .. } - )); - - let mut ambiguous = direct.clone(); - let mut conflicting = direct[0]; - conflicting.transactions_commitment = TransactionsCommitment::from_bytes([0xCD; 32]); - ambiguous.push(conflicting); - assert_eq!( - node.compare_direct_deliveries(ambiguous).unwrap(), - ShadowDeliveryComparisonV1::Ambiguous { - slots: vec![ShadowDeliverySlotV1 { - author: direct[0].author, - round: direct[0].round, - }], - } - ); - } - - #[test] - fn recovery_api_binds_requested_reference_before_model_transition() { - let mut network = TestNetwork::new(); - let candidate = round_one_candidate(1, &network.committee, 0xA1); - let mut wrong = candidate.reference(); - wrong.digest = BlockDigest::from([0xFF; 32]); - assert!(matches!( - network.nodes[0] - .recover_candidate_for(wrong, &candidate.canonical_wire_bytes().unwrap()), - Err(ShadowErrorV1::Carrier( - RbcDagError::ReferenceMismatch { .. } - )) - )); - } - - #[test] - fn recovered_content_is_durable_only_after_embedded_phase_evidence() { - let mut network = TestNetwork::new(); - let target = round_one_candidate(3, &network.committee, 0xB1); - for sender in 1..N { - let outer = round_two_phase_carrier( - sender as AuthorityIndex, - RbcPhaseStatementV1::Echo { - target: target.reference(), - }, - &network.committee, - ); - let authentication = network - .context - .authenticate_with_committee( - &outer, - &network.committee, - CarrierAuthorizerV1::MacVector { - authority: sender as AuthorityIndex, - keys: &network.keyrings[sender], - }, - ) - .unwrap(); - network.nodes[0] - .receive_authenticated_from_peer( - &outer.canonical_wire_bytes().unwrap(), - &authentication.canonical_wire_bytes(), - sender as AuthorityIndex, - ) - .unwrap(); - } - assert!( - network.nodes[0] - .retained_candidate_wire(target.reference()) - .is_none() - ); - network.nodes[0] - .recover_candidate_for(target.reference(), &target.canonical_wire_bytes().unwrap()) - .unwrap(); - assert_eq!( - network.nodes[0] - .retained_candidate_wire(target.reference()) - .unwrap(), - target.canonical_wire_bytes().unwrap() - ); - } - - fn round_one_candidate( - author: AuthorityIndex, - committee: &RbcDagCommitteeContextV1, - marker: u8, - ) -> CandidateCarrierV1 { - let weak_parents = committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(carrier_genesis_reference) - .collect(); - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 1, - own_prev: carrier_genesis_reference(author), - weak_parents, - transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: 1, - }, - committee, - ) - .unwrap() - } - - fn round_two_phase_carrier( - author: AuthorityIndex, - statement: RbcPhaseStatementV1, - committee: &RbcDagCommitteeContextV1, - ) -> CandidateCarrierV1 { - let previous = |authority: AuthorityIndex| BlockReference { - authority, - round: 1, - digest: BlockDigest::from([0xC0 + authority as u8; 32]), - }; - let weak_parents = committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(previous) - .collect(); - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 2, - own_prev: previous(author), - weak_parents, - transactions_commitment: TransactionsCommitment::from_bytes( - [0xD0 + author as u8; 32], - ), - data_acknowledgments: Vec::new(), - phase_batch: vec![statement], - consensus_vertex: None, - creation_time_ns: 2, - }, - committee, - ) - .unwrap() - } -} diff --git a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs b/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs deleted file mode 100644 index 49879cc2..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag_shadow_service.rs +++ /dev/null @@ -1,2071 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Async, non-authoritative network adapter for the persisted RBC-DAG shadow. - -use std::{ - collections::{BTreeMap, BTreeSet}, - error::Error, - fmt, - path::Path, - sync::Arc, - time::{Duration, Instant}, -}; - -use parking_lot::Mutex; - -use tokio::{ - sync::{ - mpsc::{self, error::TrySendError}, - oneshot, - }, - task::JoinHandle, -}; - -use crate::{ - crypto::{MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, SIGNATURE_SIZE}, - network::{NetworkMessage, RbcDagShadowCarrier, RbcDagShadowCarrierResponse}, - starfish_rbc::RbcCanonicalHeader, - starfish_rbc_dag::{ - MAX_CARRIER_CONTENT_SIZE_V1, RbcDagCommitteeContextV1, RbcDagContextV1, - model::{ModelEffect, ModelError}, - }, - starfish_rbc_dag_shadow::{ - ShadowAuthorizerV1, ShadowDeliveryComparisonV1, ShadowDeliveryIdentityV1, - ShadowDeliverySlotV1, ShadowErrorV1, ShadowIngressDispositionV1, ShadowOpenReportV1, - ShadowOutboundEnvelopeV1, StarfishRbcDagShadowV1, - }, - types::{AuthorityIndex, BlockAuthenticationScheme, BlockReference, RoundNumber, TimestampNs}, -}; - -// A shadow run must absorb one complete committee fan-in plus a small reserve -// for the local carrier and control notifications before its single fsync -// owner can drain. At the four-MiB carrier cap, allowing at most 64 queued -// inputs also caps carrier payload retention at 256 MiB (plus bounded -// sidecars and allocator overhead). Larger committees are rejected for this -// benchmark prototype instead of silently under-sizing the queue and -// reporting incomparable results. -// Use the full bounded allowance even for a small committee. A single fan-in -// reserve is insufficient when several round bursts arrive while the actor is -// synchronously making the previous transition durable. -const SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1: usize = 64; -const SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1: usize = 64; -const SHADOW_SERVICE_CONTROL_RESERVE_V1: usize = 5; -const SHADOW_SERVICE_EVENT_CAPACITY_V1: usize = 16; -const SHADOW_RECOVERY_RETRY_INTERVAL_V1: Duration = Duration::from_millis(500); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct ShadowLocalCarrierV1 { - author: AuthorityIndex, - round: RoundNumber, - transactions_commitment: crate::crypto::TransactionsCommitment, - creation_time_ns: TimestampNs, -} - -impl ShadowLocalCarrierV1 { - fn from_direct_header(header: &RbcCanonicalHeader) -> Self { - Self { - author: header.reference().authority, - round: header.reference().round, - transactions_commitment: header.transactions_commitment(), - creation_time_ns: header.meta_creation_time_ns(), - } - } -} - -enum ShadowServiceMessageV1 { - LocalCarrier(ShadowLocalCarrierV1), - Carrier { - peer: AuthorityIndex, - envelope: RbcDagShadowCarrier, - }, - CarrierRequest { - peer: AuthorityIndex, - reference: BlockReference, - }, - CarrierResponse { - peer: AuthorityIndex, - response: RbcDagShadowCarrierResponse, - }, - DirectDeliveriesChanged, - TopologyChanged, - RetryRecovery, - Shutdown(oneshot::Sender>), -} - -#[derive(Clone)] -pub(crate) struct StarfishRbcDagShadowServiceHandleV1 { - sender: mpsc::Sender, - max_sidecar_size: usize, - own_authority: AuthorityIndex, - committee_size: usize, - input_capacity: usize, - desired_topology: Arc>>, - desired_direct_deliveries: Arc>>, - invalidated_by_overload: Arc>>, -} - -impl StarfishRbcDagShadowServiceHandleV1 { - fn send(&self, message: ShadowServiceMessageV1) -> Result<(), ShadowServiceErrorV1> { - let kind = message.kind(); - if let Some(reason) = *self.invalidated_by_overload.lock() { - return Err(ShadowServiceErrorV1::BenchmarkInvalid { reason }); - } - self.sender.try_send(message).map_err(|error| match error { - TrySendError::Full(_) => { - *self.invalidated_by_overload.lock() = Some(kind); - ShadowServiceErrorV1::Overloaded { - kind, - capacity: self.input_capacity, - } - } - TrySendError::Closed(_) => ShadowServiceErrorV1::Stopped, - }) - } - - pub(crate) fn local_header( - &self, - header: &RbcCanonicalHeader, - ) -> Result<(), ShadowServiceErrorV1> { - self.send(ShadowServiceMessageV1::LocalCarrier( - ShadowLocalCarrierV1::from_direct_header(header), - )) - } - - pub(crate) fn carrier( - &self, - peer: AuthorityIndex, - envelope: RbcDagShadowCarrier, - ) -> Result<(), ShadowServiceErrorV1> { - validate_wire_size( - "carrier", - envelope.canonical_carrier.len(), - MAX_CARRIER_CONTENT_SIZE_V1, - )?; - validate_wire_size( - "authentication sidecar", - envelope.authentication_sidecar.len(), - self.max_sidecar_size, - )?; - self.send(ShadowServiceMessageV1::Carrier { peer, envelope }) - } - - pub(crate) fn carrier_request( - &self, - peer: AuthorityIndex, - reference: BlockReference, - ) -> Result<(), ShadowServiceErrorV1> { - self.send(ShadowServiceMessageV1::CarrierRequest { peer, reference }) - } - - pub(crate) fn carrier_response( - &self, - peer: AuthorityIndex, - response: RbcDagShadowCarrierResponse, - ) -> Result<(), ShadowServiceErrorV1> { - validate_wire_size( - "carrier response", - response.canonical_carrier.len(), - MAX_CARRIER_CONTENT_SIZE_V1, - )?; - self.send(ShadowServiceMessageV1::CarrierResponse { peer, response }) - } - - pub(crate) fn direct_delivered( - &self, - identity: ShadowDeliveryIdentityV1, - ) -> Result<(), ShadowServiceErrorV1> { - if identity.author as usize >= self.committee_size { - return Err(ShadowServiceErrorV1::UnknownAuthority(identity.author)); - } - if !self.desired_direct_deliveries.lock().insert(identity) { - return Ok(()); - } - match self - .sender - .try_send(ShadowServiceMessageV1::DirectDeliveriesChanged) - { - Ok(()) | Err(TrySendError::Full(_)) => Ok(()), - Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), - } - } - - pub(crate) fn peer_connected(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { - self.update_peer(peer, true) - } - - pub(crate) fn peer_disconnected( - &self, - peer: AuthorityIndex, - ) -> Result<(), ShadowServiceErrorV1> { - self.update_peer(peer, false) - } - - pub(crate) async fn shutdown(&self) -> Result<(), ShadowServiceErrorV1> { - let (reply, receiver) = oneshot::channel(); - self.sender - .send(ShadowServiceMessageV1::Shutdown(reply)) - .await - .map_err(|_| ShadowServiceErrorV1::Stopped)?; - receiver.await.map_err(|_| ShadowServiceErrorV1::Stopped)? - } - - fn update_peer( - &self, - peer: AuthorityIndex, - connected: bool, - ) -> Result<(), ShadowServiceErrorV1> { - if peer as usize >= self.committee_size { - return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); - } - if peer == self.own_authority { - return Err(ShadowServiceErrorV1::Loopback(peer)); - } - let mut topology = self.desired_topology.lock(); - let state = topology.entry(peer).or_insert((false, 0)); - if state.0 != connected { - state.0 = connected; - state.1 = state.1.saturating_add(1); - } - drop(topology); - match self - .sender - .try_send(ShadowServiceMessageV1::TopologyChanged) - { - Ok(()) | Err(TrySendError::Full(_)) => Ok(()), - Err(TrySendError::Closed(_)) => Err(ShadowServiceErrorV1::Stopped), - } - } -} - -impl ShadowServiceMessageV1 { - fn kind(&self) -> &'static str { - match self { - Self::LocalCarrier(_) => "local", - Self::Carrier { .. } => "carrier", - Self::CarrierRequest { .. } => "carrier_request", - Self::CarrierResponse { .. } => "carrier_response", - Self::DirectDeliveriesChanged => "direct_deliveries_changed", - Self::TopologyChanged => "topology_changed", - Self::RetryRecovery => "recovery_retry", - Self::Shutdown(_) => "shutdown", - } - } -} - -#[derive(Debug)] -pub(crate) enum ShadowServiceEventV1 { - Ready, - ComparisonBacklog { - unpaired_direct: usize, - unpaired_shadow: usize, - max_round_lag: RoundNumber, - }, - Network { - recipient: AuthorityIndex, - message: NetworkMessage, - }, - Delivered(ShadowDeliveryIdentityV1), - Comparison(ShadowDeliveryComparisonV1), - Input { - kind: &'static str, - outcome: &'static str, - }, - WalDurable { - batches: u64, - records: u64, - }, - Recovered { - batches: u64, - discarded_tail_bytes: u64, - }, - PendingRecovery(usize), - Rejected { - peer: Option, - error: String, - }, -} - -#[derive(Debug)] -pub(crate) enum ShadowServiceErrorV1 { - Shadow(ShadowErrorV1), - StartTask(tokio::task::JoinError), - Stopped, - Overloaded { - kind: &'static str, - capacity: usize, - }, - BenchmarkInvalid { - reason: &'static str, - }, - InputTooLarge { - field: &'static str, - actual: usize, - maximum: usize, - }, - CommitteeBurstTooLarge { - committee_size: usize, - maximum_capacity: usize, - }, - UnknownAuthority(AuthorityIndex), - Loopback(AuthorityIndex), - ConflictingLocalHeader(RoundNumber), - MissingRecoveredLocalHeader(RoundNumber), - RecoveredLocalHeaderMismatch(RoundNumber), - LocalHeaderAuthority { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - UnauthenticatedCarrierRetained, - UnexpectedResponse(BlockReference), - ResponseFromNonHolder { - peer: AuthorityIndex, - reference: BlockReference, - }, -} - -impl fmt::Display for ShadowServiceErrorV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Shadow(error) => error.fmt(formatter), - Self::StartTask(error) => write!( - formatter, - "Starfish-RBC-DAG shadow startup task failed: {error}" - ), - Self::Stopped => formatter.write_str("Starfish-RBC-DAG shadow service stopped"), - Self::Overloaded { kind, capacity } => write!( - formatter, - "Starfish-RBC-DAG shadow {kind} input was dropped because the queue is full \ - (capacity {capacity}); benchmark comparison is invalid" - ), - Self::BenchmarkInvalid { reason } => write!( - formatter, - "Starfish-RBC-DAG shadow benchmark was disabled after dropping {reason} input" - ), - Self::InputTooLarge { - field, - actual, - maximum, - } => write!( - formatter, - "Starfish-RBC-DAG shadow {field} is {actual} bytes, above the {maximum}-byte limit" - ), - Self::CommitteeBurstTooLarge { - committee_size, - maximum_capacity, - } => write!( - formatter, - "Starfish-RBC-DAG shadow committee size {committee_size} needs a burst queue of \ - {}, above the memory-safe capacity limit {maximum_capacity}", - committee_size - .saturating_sub(1) - .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1), - ), - Self::UnknownAuthority(authority) => { - write!(formatter, "unknown shadow peer authority {authority}") - } - Self::Loopback(authority) => { - write!(formatter, "shadow peer authority {authority} is local") - } - Self::ConflictingLocalHeader(round) => write!( - formatter, - "conflicting direct headers supplied for queued shadow round {round}" - ), - Self::MissingRecoveredLocalHeader(round) => write!( - formatter, - "persisted shadow carrier at round {round} has no matching recovered direct header" - ), - Self::RecoveredLocalHeaderMismatch(round) => write!( - formatter, - "persisted shadow carrier and recovered direct header disagree at round {round}" - ), - Self::LocalHeaderAuthority { expected, actual } => write!( - formatter, - "shadow local header authority {actual} does not match local authority {expected}" - ), - Self::UnauthenticatedCarrierRetained => formatter.write_str( - "shadow carrier authentication failed; canonical content was retained candidate-only", - ), - Self::UnexpectedResponse(reference) => { - write!(formatter, "unexpected shadow response for {reference}") - } - Self::ResponseFromNonHolder { peer, reference } => write!( - formatter, - "shadow response for {reference} came from non-holder {peer}" - ), - } - } -} - -impl Error for ShadowServiceErrorV1 { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Shadow(error) => Some(error), - Self::StartTask(error) => Some(error), - _ => None, - } - } -} - -impl From for ShadowServiceErrorV1 { - fn from(error: ShadowErrorV1) -> Self { - Self::Shadow(error) - } -} - -pub(crate) fn start_starfish_rbc_dag_shadow_service_v1( - path: impl AsRef, - committee: RbcDagCommitteeContextV1, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - authorizer: ShadowAuthorizerV1, - recovered_local_headers: Vec, -) -> Result< - ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ), - ShadowServiceErrorV1, -> { - let committee_size = committee.committee().len(); - let input_capacity = shadow_input_capacity(committee_size)?; - let max_sidecar_size = - authentication_sidecar_size(context.authentication_scheme(), committee_size); - let path = path.as_ref().to_path_buf(); - let mut pending_local = BTreeMap::new(); - for header in recovered_local_headers { - let local = ShadowLocalCarrierV1::from_direct_header(&header); - if local.author != own_authority { - return Err(ShadowServiceErrorV1::LocalHeaderAuthority { - expected: own_authority, - actual: local.author, - }); - } - if let Some(previous) = pending_local.insert(local.round, local) { - if previous != local { - return Err(ShadowServiceErrorV1::ConflictingLocalHeader(local.round)); - } - } - } - let (message_tx, message_rx) = mpsc::channel(input_capacity); - let (event_tx, event_rx) = mpsc::channel(SHADOW_SERVICE_EVENT_CAPACITY_V1); - let desired_topology = Arc::new(Mutex::new(BTreeMap::new())); - let desired_direct_deliveries = Arc::new(Mutex::new(BTreeSet::new())); - let invalidated_by_overload = Arc::new(Mutex::new(None)); - let retry_tx = message_tx.downgrade(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(SHADOW_RECOVERY_RETRY_INTERVAL_V1); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - interval.tick().await; - loop { - interval.tick().await; - let Some(retry_tx) = retry_tx.upgrade() else { - break; - }; - match retry_tx.try_send(ShadowServiceMessageV1::RetryRecovery) { - Ok(()) | Err(TrySendError::Full(_)) => {} - Err(TrySendError::Closed(_)) => break, - } - } - }); - let startup_events = event_tx.clone(); - let actor_desired_topology = Arc::clone(&desired_topology); - let actor_desired_direct_deliveries = Arc::clone(&desired_direct_deliveries); - let actor_invalidated_by_overload = Arc::clone(&invalidated_by_overload); - let task = tokio::spawn(async move { - let opened = tokio::task::spawn_blocking(move || { - StarfishRbcDagShadowV1::open(path, committee, own_authority, context, authorizer) - }) - .await; - let (core, open_report) = match opened { - Ok(Ok(opened)) => opened, - Ok(Err(error)) => { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: error.to_string(), - }) - .await; - return; - } - Err(error) => { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::StartTask(error).to_string(), - }) - .await; - return; - } - }; - let persisted_local = match core.local_outbound_metadata() { - Ok(metadata) => metadata - .into_iter() - .map(|(round, commitment, creation_time_ns)| { - (round, (commitment, creation_time_ns)) - }) - .collect::>(), - Err(error) => { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: error.to_string(), - }) - .await; - return; - } - }; - let durable_round = core.local_carrier_round(); - for (round, (commitment, creation_time_ns)) in &persisted_local { - let Some(recovered) = pending_local.get(round) else { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::MissingRecoveredLocalHeader(*round) - .to_string(), - }) - .await; - return; - }; - if recovered.transactions_commitment != *commitment - || recovered.creation_time_ns != *creation_time_ns - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(*round) - .to_string(), - }) - .await; - return; - } - } - if let Some(round) = pending_local - .keys() - .copied() - .find(|round| *round < durable_round && !persisted_local.contains_key(round)) - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: ShadowServiceErrorV1::RecoveredLocalHeaderMismatch(round).to_string(), - }) - .await; - return; - } - let reported_shadow_deliveries = match core.delivered_identities() { - Ok(identities) => identities.into_iter().collect::>(), - Err(error) => { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: error.to_string(), - }) - .await; - return; - } - }; - let recovered_shadow_deliveries = reported_shadow_deliveries.clone(); - let reported_shadow_delivery_slots = reported_shadow_deliveries - .iter() - .map(delivery_slot) - .collect(); - let comparison_backlog = ShadowComparisonBacklogV1::new(reported_shadow_delivery_slots); - pending_local.retain(|round, _| *round >= core.local_carrier_round()); - let state = ShadowServiceStateV1 { - core, - own_authority, - committee_size, - events: event_tx, - connected: BTreeSet::new(), - desired_topology: actor_desired_topology, - desired_direct_deliveries: actor_desired_direct_deliveries, - observed_topology: BTreeMap::new(), - invalidated_by_overload: actor_invalidated_by_overload, - pending_local, - pending_recovery: BTreeMap::new(), - recovery_last_attempt: BTreeMap::new(), - direct_deliveries: BTreeSet::new(), - reported_shadow_deliveries, - recovered_shadow_deliveries, - comparison_backlog, - reported_matches: BTreeSet::new(), - reported_mismatches: BTreeSet::new(), - reported_conflicts: BTreeSet::new(), - fatal: false, - }; - if let Err(error) = tokio::task::spawn_blocking(move || { - run_shadow_service(state, message_rx, open_report); - }) - .await - { - let _ = startup_events - .send(ShadowServiceEventV1::Rejected { - peer: None, - error: format!("Starfish-RBC-DAG shadow actor task failed: {error}"), - }) - .await; - } - }); - Ok(( - StarfishRbcDagShadowServiceHandleV1 { - sender: message_tx, - max_sidecar_size, - own_authority, - committee_size, - input_capacity, - desired_topology, - desired_direct_deliveries, - invalidated_by_overload, - }, - event_rx, - task, - )) -} - -struct ShadowComparisonBacklogV1 { - direct_slots: BTreeSet, - reported_shadow_slots: BTreeSet, - // Recovered shadow slots are deliberately absent from this set: they may - // pair a current direct observation but are not current-process timing - // observations and therefore cannot create shadow-only backlog. - epoch_shadow_slots: BTreeSet, - unpaired_direct_slots: BTreeSet, - unpaired_shadow_slots: BTreeSet, - latest_epoch_round: RoundNumber, -} - -impl ShadowComparisonBacklogV1 { - fn new(reported_shadow_slots: BTreeSet) -> Self { - Self { - direct_slots: BTreeSet::new(), - reported_shadow_slots, - epoch_shadow_slots: BTreeSet::new(), - unpaired_direct_slots: BTreeSet::new(), - unpaired_shadow_slots: BTreeSet::new(), - latest_epoch_round: 0, - } - } - - fn observe_direct(&mut self, slot: ShadowDeliverySlotV1) { - self.observe_epoch_round(slot); - if self.direct_slots.insert(slot) { - if !self.reported_shadow_slots.contains(&slot) { - self.unpaired_direct_slots.insert(slot); - } - self.unpaired_shadow_slots.remove(&slot); - } - } - - fn observe_epoch_shadow(&mut self, slot: ShadowDeliverySlotV1) { - self.observe_epoch_round(slot); - self.reported_shadow_slots.insert(slot); - if self.epoch_shadow_slots.insert(slot) { - if !self.direct_slots.contains(&slot) { - self.unpaired_shadow_slots.insert(slot); - } - self.unpaired_direct_slots.remove(&slot); - } - } - - fn observe_epoch_round(&mut self, slot: ShadowDeliverySlotV1) { - self.latest_epoch_round = self.latest_epoch_round.max(slot.round); - } - - fn counts(&self) -> (usize, usize, RoundNumber) { - let max_round_lag = self - .unpaired_direct_slots - .union(&self.unpaired_shadow_slots) - .map(|slot| self.latest_epoch_round.saturating_sub(slot.round)) - .max() - .unwrap_or(0); - ( - self.unpaired_direct_slots.len(), - self.unpaired_shadow_slots.len(), - max_round_lag, - ) - } -} - -struct ShadowServiceStateV1 { - core: StarfishRbcDagShadowV1, - own_authority: AuthorityIndex, - committee_size: usize, - events: mpsc::Sender, - connected: BTreeSet, - desired_topology: Arc>>, - desired_direct_deliveries: Arc>>, - observed_topology: BTreeMap, - invalidated_by_overload: Arc>>, - pending_local: BTreeMap, - pending_recovery: BTreeMap>, - recovery_last_attempt: BTreeMap<(BlockReference, AuthorityIndex), Instant>, - direct_deliveries: BTreeSet, - reported_shadow_deliveries: BTreeSet, - recovered_shadow_deliveries: BTreeSet, - comparison_backlog: ShadowComparisonBacklogV1, - reported_matches: BTreeSet, - reported_mismatches: BTreeSet<(ShadowDeliveryIdentityV1, ShadowDeliveryIdentityV1)>, - reported_conflicts: BTreeSet, - fatal: bool, -} - -impl ShadowServiceStateV1 { - fn emit(&self, event: ShadowServiceEventV1) { - let _ = self.events.blocking_send(event); - } - - fn reject(&self, peer: Option, error: impl fmt::Display) { - self.emit(ShadowServiceEventV1::Rejected { - peer, - error: error.to_string(), - }); - } - - fn emit_comparison_backlog(&self) { - let (unpaired_direct, unpaired_shadow, max_round_lag) = self.comparison_backlog.counts(); - self.emit(ShadowServiceEventV1::ComparisonBacklog { - unpaired_direct, - unpaired_shadow, - max_round_lag, - }); - } - - fn validate_peer(&self, peer: AuthorityIndex) -> Result<(), ShadowServiceErrorV1> { - if peer as usize >= self.committee_size { - return Err(ShadowServiceErrorV1::UnknownAuthority(peer)); - } - if peer == self.own_authority { - return Err(ShadowServiceErrorV1::Loopback(peer)); - } - Ok(()) - } - - fn reconcile_topology(&mut self) { - let desired = self.desired_topology.lock().clone(); - let mut newly_connected = Vec::new(); - for (peer, state) in &desired { - if self.observed_topology.get(peer) == Some(state) { - continue; - } - self.recovery_last_attempt - .retain(|(_, holder), _| holder != peer); - if state.0 { - self.connected.insert(*peer); - newly_connected.push(*peer); - } else { - self.connected.remove(peer); - } - } - self.observed_topology = desired; - if !newly_connected.is_empty() { - let retransmissions = self.core.retransmissions(); - for peer in newly_connected { - for envelope in &retransmissions { - self.send_envelope(peer, envelope); - } - } - self.flush_recovery_requests(); - } - } - - fn reconcile_direct_deliveries(&mut self) { - let desired = self.desired_direct_deliveries.lock().clone(); - let newly_observed = desired - .difference(&self.direct_deliveries) - .copied() - .collect::>(); - for identity in newly_observed { - self.direct_deliveries.insert(identity); - let slot = delivery_slot(&identity); - self.comparison_backlog.observe_direct(slot); - self.emit(ShadowServiceEventV1::Input { - kind: "delivery", - outcome: "direct", - }); - self.emit_slot_comparison(slot); - } - if !desired.is_empty() { - self.emit_comparison_backlog(); - } - } - - fn observe_external_invalidation(&mut self) -> bool { - let reason = *self.invalidated_by_overload.lock(); - if let Some(reason) = reason { - self.mark_fatal(ShadowServiceErrorV1::BenchmarkInvalid { reason }); - true - } else { - false - } - } - - fn mark_fatal(&mut self, error: impl fmt::Display) { - if self.fatal { - return; - } - self.fatal = true; - if self.invalidated_by_overload.lock().is_none() { - *self.invalidated_by_overload.lock() = Some("fatal_core"); - } - self.emit(ShadowServiceEventV1::Input { - kind: "benchmark", - outcome: "invalid", - }); - self.reject(None, error); - } - - fn broadcast(&self, envelope: &ShadowOutboundEnvelopeV1) { - for recipient in 0..self.committee_size { - let recipient = recipient as AuthorityIndex; - if recipient != self.own_authority { - self.send_envelope(recipient, envelope); - } - } - } - - fn send_envelope(&self, recipient: AuthorityIndex, envelope: &ShadowOutboundEnvelopeV1) { - self.emit(ShadowServiceEventV1::Network { - recipient, - message: NetworkMessage::RbcDagShadowCarrier(RbcDagShadowCarrier { - canonical_carrier: envelope.canonical_carrier_wire().to_vec(), - authentication_sidecar: envelope.authentication_sidecar().to_vec(), - }), - }); - } - - fn report_wal_delta(&self, before: (u64, u64)) { - let after = self.core.wal_counts(); - let batches = after.0.saturating_sub(before.0); - let records = after.1.saturating_sub(before.1); - if batches != 0 || records != 0 { - self.emit(ShadowServiceEventV1::WalDurable { batches, records }); - } - } - - fn process_effects(&mut self, effects: Vec) { - for effect in effects { - match effect { - ModelEffect::NeedCarrier { target, holders } => { - self.pending_recovery.entry(target).or_default().extend( - holders.into_iter().filter(|holder| { - *holder != self.own_authority - && (*holder as usize) < self.committee_size - }), - ); - } - ModelEffect::Delivered(reference) => { - self.pending_recovery.remove(&reference); - } - ModelEffect::PrefixAdvanced { .. } | ModelEffect::CarrierRoundAdvanced(_) => {} - } - } - self.reconcile_pending_recovery(); - self.flush_recovery_requests(); - self.emit(ShadowServiceEventV1::PendingRecovery( - self.pending_recovery.len(), - )); - self.report_new_shadow_deliveries(); - } - - fn reconcile_pending_recovery(&mut self) { - self.pending_recovery - .retain(|target, _| self.core.retained_candidate_wire(*target).is_none()); - self.recovery_last_attempt.retain(|(target, holder), _| { - self.pending_recovery - .get(target) - .is_some_and(|holders| holders.contains(holder)) - }); - } - - fn enqueue_local(&mut self, local: ShadowLocalCarrierV1) { - if local.author != self.own_authority { - self.reject( - None, - ShadowServiceErrorV1::LocalHeaderAuthority { - expected: self.own_authority, - actual: local.author, - }, - ); - return; - } - let durable_round = self.core.local_carrier_round(); - if local.round < durable_round { - self.emit(ShadowServiceEventV1::Input { - kind: "local", - outcome: "already_durable", - }); - return; - } - if let Some(existing) = self.pending_local.get(&local.round) { - if existing == &local { - self.emit(ShadowServiceEventV1::Input { - kind: "local", - outcome: "duplicate", - }); - } else { - self.reject( - None, - ShadowServiceErrorV1::ConflictingLocalHeader(local.round), - ); - } - return; - } - let round = local.round; - self.pending_local.insert(round, local); - if round != durable_round || !self.core.can_create_carrier() { - self.emit(ShadowServiceEventV1::Input { - kind: "local", - outcome: "queued", - }); - } - self.retry_pending_local(); - } - - /// Create only the exact round opened by the durable carrier clock. A - /// future direct header remains queued until authenticated carrier input - /// advances the model; recovered historical headers below that clock are - /// harmless idempotent replays. - fn retry_pending_local(&mut self) { - loop { - let durable_round = self.core.local_carrier_round(); - self.pending_local - .retain(|round, _| *round >= durable_round); - if !self.core.can_create_carrier() { - return; - } - let Some(local) = self.pending_local.remove(&durable_round) else { - return; - }; - let before = self.core.wal_counts(); - match self.core.create_local_carrier( - local.round, - local.transactions_commitment, - local.creation_time_ns, - ) { - Ok((envelope, effects)) => { - self.emit(ShadowServiceEventV1::Input { - kind: "local", - outcome: "accepted", - }); - self.report_wal_delta(before); - self.broadcast(&envelope); - self.process_effects(effects); - } - Err(ShadowErrorV1::Model(ModelError::LocalRoundNotOpen(_))) => { - self.pending_local.insert(local.round, local); - self.emit(ShadowServiceEventV1::Input { - kind: "local", - outcome: "queued", - }); - return; - } - Err(error) => { - self.emit(ShadowServiceEventV1::Input { - kind: "local", - outcome: "rejected", - }); - self.mark_fatal(error); - return; - } - } - } - } - - fn flush_recovery_requests(&mut self) { - let now = Instant::now(); - let mut requests = Vec::new(); - for (reference, holders) in &self.pending_recovery { - for holder in holders { - if self.connected.contains(holder) - && self - .recovery_last_attempt - .get(&(*reference, *holder)) - .is_none_or(|last| { - now.saturating_duration_since(*last) - >= SHADOW_RECOVERY_RETRY_INTERVAL_V1 - }) - { - requests.push((*reference, *holder)); - } - } - } - for (reference, holder) in requests { - self.recovery_last_attempt.insert((reference, holder), now); - self.emit(ShadowServiceEventV1::Network { - recipient: holder, - message: NetworkMessage::RbcDagShadowCarrierRequest(reference), - }); - } - } - - fn report_new_shadow_deliveries(&mut self) { - let identities: BTreeSet<_> = match self.core.delivered_identities() { - Ok(identities) => identities.into_iter().collect(), - Err(error) => { - self.reject(None, error); - return; - } - }; - let new_identities = identities - .difference(&self.reported_shadow_deliveries) - .copied() - .collect::>(); - self.reported_shadow_deliveries = identities; - for identity in &new_identities { - let slot = delivery_slot(identity); - self.comparison_backlog.observe_epoch_shadow(slot); - self.emit(ShadowServiceEventV1::Delivered(*identity)); - self.emit_slot_comparison(slot); - self.emit_comparison_backlog(); - } - } - - fn emit_slot_comparison(&mut self, slot: ShadowDeliverySlotV1) { - let direct = self - .direct_deliveries - .iter() - .filter(|identity| delivery_slot(identity) == slot) - .copied() - .collect::>(); - let shadow = self - .reported_shadow_deliveries - .iter() - .filter(|identity| delivery_slot(identity) == slot) - .copied() - .collect::>(); - if direct.is_empty() || shadow.is_empty() { - return; - } - if direct.len() > 1 || shadow.len() > 1 { - if self.reported_conflicts.insert(slot) { - self.emit(ShadowServiceEventV1::Comparison( - ShadowDeliveryComparisonV1::Ambiguous { slots: vec![slot] }, - )); - } - return; - } - let direct = direct[0]; - let shadow = shadow[0]; - if direct == shadow { - if self.reported_matches.insert(direct) { - if self.recovered_shadow_deliveries.contains(&shadow) { - self.emit(ShadowServiceEventV1::Input { - kind: "comparison", - outcome: "recovered_match", - }); - } - self.emit(ShadowServiceEventV1::Comparison( - ShadowDeliveryComparisonV1::Match, - )); - } - } else if self.reported_mismatches.insert((direct, shadow)) { - self.emit(ShadowServiceEventV1::Comparison( - ShadowDeliveryComparisonV1::Mismatch { - direct_only: vec![direct], - shadow_only: vec![shadow], - }, - )); - } - } -} - -fn run_shadow_service( - mut state: ShadowServiceStateV1, - mut messages: mpsc::Receiver, - open_report: ShadowOpenReportV1, -) { - if open_report.replayed_batches() != 0 || open_report.discarded_tail_bytes() != 0 { - state.emit(ShadowServiceEventV1::Recovered { - batches: open_report.replayed_batches(), - discarded_tail_bytes: open_report.discarded_tail_bytes(), - }); - } - state.reconcile_topology(); - state.reconcile_direct_deliveries(); - if !state.observe_external_invalidation() { - state.emit(ShadowServiceEventV1::Ready); - state.emit_comparison_backlog(); - state.process_effects(open_report.recovery_effects().to_vec()); - state.retry_pending_local(); - } - - while !state.fatal { - let Some(message) = messages.blocking_recv() else { - break; - }; - let message = match message { - ShadowServiceMessageV1::Shutdown(reply) => { - let events = state.events.clone(); - let result = state - .core - .shutdown() - .map(|_| ()) - .map_err(ShadowServiceErrorV1::from); - if reply.send(result).is_err() { - let _ = events.blocking_send(ShadowServiceEventV1::Rejected { - peer: None, - error: "shadow shutdown acknowledgment receiver was dropped".to_owned(), - }); - } - return; - } - message => message, - }; - if state.observe_external_invalidation() { - break; - } - match message { - ShadowServiceMessageV1::LocalCarrier(local) => { - state.enqueue_local(local); - } - ShadowServiceMessageV1::Carrier { peer, envelope } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - let before = state.core.wal_counts(); - match state.core.receive_or_retain_from_peer( - &envelope.canonical_carrier, - &envelope.authentication_sidecar, - peer, - ) { - Ok(outcome) => { - let outcome_label = match outcome.disposition() { - ShadowIngressDispositionV1::Authenticated => "authenticated", - ShadowIngressDispositionV1::CandidateRetained => { - "retained_unauthenticated" - } - ShadowIngressDispositionV1::IgnoredDuplicateConflictOrStale => { - "ignored" - } - }; - state.emit(ShadowServiceEventV1::Input { - kind: "carrier", - outcome: outcome_label, - }); - state.report_wal_delta(before); - state.process_effects(outcome.effects().to_vec()); - state.retry_pending_local(); - if outcome.disposition() == ShadowIngressDispositionV1::CandidateRetained { - state.reject( - Some(peer), - ShadowServiceErrorV1::UnauthenticatedCarrierRetained, - ); - } - } - Err(error) => { - state.emit(ShadowServiceEventV1::Input { - kind: "carrier", - outcome: "rejected", - }); - if is_fatal_core_error(&error) { - state.mark_fatal(error); - } else { - state.reject(Some(peer), error); - } - } - } - } - ShadowServiceMessageV1::CarrierRequest { peer, reference } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - if let Some(canonical_carrier) = state.core.retained_candidate_wire(reference) { - state.emit(ShadowServiceEventV1::Network { - recipient: peer, - message: NetworkMessage::RbcDagShadowCarrierResponse( - RbcDagShadowCarrierResponse { - reference, - canonical_carrier, - }, - ), - }); - } - } - ShadowServiceMessageV1::CarrierResponse { peer, response } => { - if let Err(error) = state.validate_peer(peer) { - state.reject(Some(peer), error); - continue; - } - let Some(holders) = state.pending_recovery.get(&response.reference) else { - state.reject( - Some(peer), - ShadowServiceErrorV1::UnexpectedResponse(response.reference), - ); - continue; - }; - if !holders.contains(&peer) { - state.reject( - Some(peer), - ShadowServiceErrorV1::ResponseFromNonHolder { - peer, - reference: response.reference, - }, - ); - continue; - } - let before = state.core.wal_counts(); - match state - .core - .recover_candidate_for(response.reference, &response.canonical_carrier) - { - Ok(effects) => { - state.pending_recovery.remove(&response.reference); - state - .recovery_last_attempt - .retain(|(target, _), _| *target != response.reference); - state.emit(ShadowServiceEventV1::Input { - kind: "recovery", - outcome: "accepted", - }); - state.report_wal_delta(before); - state.process_effects(effects); - state.retry_pending_local(); - } - Err(error) => { - state.emit(ShadowServiceEventV1::Input { - kind: "recovery", - outcome: "rejected", - }); - if is_fatal_core_error(&error) { - state.mark_fatal(error); - } else { - state.reject(Some(peer), error); - } - } - } - } - ShadowServiceMessageV1::DirectDeliveriesChanged => { - state.reconcile_direct_deliveries(); - } - ShadowServiceMessageV1::TopologyChanged => state.reconcile_topology(), - ShadowServiceMessageV1::RetryRecovery => { - state.reconcile_topology(); - state.reconcile_pending_recovery(); - state.flush_recovery_requests(); - } - ShadowServiceMessageV1::Shutdown(_) => unreachable!("shutdown handled before dispatch"), - } - state.reconcile_topology(); - state.reconcile_direct_deliveries(); - } - let events = state.events.clone(); - if let Err(error) = state.core.shutdown() { - let _ = events.blocking_send(ShadowServiceEventV1::Rejected { - peer: None, - error: error.to_string(), - }); - } -} - -fn delivery_slot(identity: &ShadowDeliveryIdentityV1) -> ShadowDeliverySlotV1 { - ShadowDeliverySlotV1 { - author: identity.author, - round: identity.round, - } -} - -fn authentication_sidecar_size(scheme: BlockAuthenticationScheme, committee_size: usize) -> usize { - const SIDECAR_HEADER_SIZE: usize = 3; - SIDECAR_HEADER_SIZE - + match scheme { - BlockAuthenticationScheme::Ed25519 => SIGNATURE_SIZE, - BlockAuthenticationScheme::MlDsa44 => ML_DSA_44_SIGNATURE_SIZE, - BlockAuthenticationScheme::MlDsa65 => ML_DSA_65_SIGNATURE_SIZE, - BlockAuthenticationScheme::MacVector => committee_size.saturating_mul(MAC_TAG_SIZE), - } -} - -fn shadow_input_capacity(committee_size: usize) -> Result { - let committee_burst = committee_size - .saturating_sub(1) - .saturating_add(SHADOW_SERVICE_CONTROL_RESERVE_V1); - if committee_burst > SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1 { - return Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { - committee_size, - maximum_capacity: SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1, - }); - } - Ok(committee_burst.max(SHADOW_SERVICE_MIN_INPUT_CAPACITY_V1)) -} - -fn validate_wire_size( - field: &'static str, - actual: usize, - maximum: usize, -) -> Result<(), ShadowServiceErrorV1> { - if actual > maximum { - Err(ShadowServiceErrorV1::InputTooLarge { - field, - actual, - maximum, - }) - } else { - Ok(()) - } -} - -fn is_fatal_core_error(error: &ShadowErrorV1) -> bool { - matches!( - error, - ShadowErrorV1::Wal(_) - | ShadowErrorV1::PostDurabilityCommit(_) - | ShadowErrorV1::PostDurabilityJournal(_) - | ShadowErrorV1::Poisoned - ) -} - -#[cfg(test)] -mod tests { - use std::{sync::Arc, time::Duration}; - - use tempfile::TempDir; - use tokio::time::timeout; - - use super::*; - use crate::{ - committee::Committee, - crypto::{TransactionsCommitment, mac_keyrings_for_test}, - starfish_rbc_dag::{ - CandidateCarrierV1, CarrierAuthorizerV1, CarrierHeaderV1Args, RbcDagProtocolInstanceId, - RbcPhaseStatementV1, carrier_genesis_reference, - }, - types::{BlockDigest, VerifiedBlock}, - }; - - const N: usize = 4; - const EVENT_TIMEOUT: Duration = Duration::from_secs(5); - - struct Harness { - _directory: TempDir, - committee: RbcDagCommitteeContextV1, - context: RbcDagContextV1, - keyrings: Vec>, - paths: Vec, - } - - impl Harness { - fn new() -> Self { - let committee = Committee::new_test(vec![1; N]); - let committee = RbcDagCommitteeContextV1::new(Arc::clone(&committee)).unwrap(); - let context = RbcDagContextV1::new_with_committee( - RbcDagProtocolInstanceId::new([0xD7; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ); - let directory = tempfile::tempdir().unwrap(); - let paths = (0..N) - .map(|authority| directory.path().join(format!("shadow-{authority}.wal"))) - .collect(); - Self { - _directory: directory, - committee, - context, - keyrings: mac_keyrings_for_test(N), - paths, - } - } - - fn start( - &self, - authority: AuthorityIndex, - recovered: Vec, - ) -> ( - StarfishRbcDagShadowServiceHandleV1, - mpsc::Receiver, - JoinHandle<()>, - ) { - start_starfish_rbc_dag_shadow_service_v1( - &self.paths[authority as usize], - self.committee.clone(), - authority, - self.context, - ShadowAuthorizerV1::MacVector(self.keyrings[authority as usize].clone()), - recovered, - ) - .unwrap() - } - - fn envelope( - &self, - candidate: &CandidateCarrierV1, - author: AuthorityIndex, - ) -> RbcDagShadowCarrier { - let authentication = self - .context - .authenticate_with_committee( - candidate, - &self.committee, - CarrierAuthorizerV1::MacVector { - authority: author, - keys: &self.keyrings[author as usize], - }, - ) - .unwrap(); - RbcDagShadowCarrier { - canonical_carrier: candidate.canonical_wire_bytes().unwrap(), - authentication_sidecar: authentication.canonical_wire_bytes(), - } - } - } - - async fn next_event(events: &mut mpsc::Receiver) -> ShadowServiceEventV1 { - timeout(EVENT_TIMEOUT, events.recv()) - .await - .expect("shadow actor timed out") - .expect("shadow actor stopped") - } - - async fn wait_ready(events: &mut mpsc::Receiver) { - loop { - match next_event(events).await { - ShadowServiceEventV1::Ready => return, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("shadow startup failed: {error}") - } - _ => {} - } - } - } - - async fn wait_backlog( - events: &mut mpsc::Receiver, - expected: (usize, usize, RoundNumber), - ) { - loop { - match next_event(events).await { - ShadowServiceEventV1::ComparisonBacklog { - unpaired_direct, - unpaired_shadow, - max_round_lag, - } if (unpaired_direct, unpaired_shadow, max_round_lag) == expected => return, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("shadow service rejected input while waiting for backlog: {error}") - } - _ => {} - } - } - } - - async fn next_carrier( - events: &mut mpsc::Receiver, - expected_recipient: AuthorityIndex, - ) -> RbcDagShadowCarrier { - loop { - if let ShadowServiceEventV1::Network { - recipient, - message: NetworkMessage::RbcDagShadowCarrier(envelope), - } = next_event(events).await - { - if recipient == expected_recipient { - return envelope; - } - } - } - } - - async fn stop( - handle: StarfishRbcDagShadowServiceHandleV1, - events: mpsc::Receiver, - task: JoinHandle<()>, - ) { - drop(events); - handle.shutdown().await.unwrap(); - task.await.unwrap(); - } - - async fn startup_rejection(mut events: mpsc::Receiver) -> String { - loop { - match next_event(&mut events).await { - ShadowServiceEventV1::Rejected { peer: None, error } => return error, - ShadowServiceEventV1::Ready => panic!("invalid shadow startup became ready"), - _ => {} - } - } - } - - fn direct_header(author: AuthorityIndex, round: RoundNumber, marker: u8) -> RbcCanonicalHeader { - RbcCanonicalHeader::try_new( - author, - round, - (0..N) - .map(|authority| { - *VerifiedBlock::new_genesis(authority as AuthorityIndex).reference() - }) - .collect(), - Vec::new(), - u64::from(round) * 1_000 + u64::from(marker), - TransactionsCommitment::from_bytes([marker; 32]), - ) - .unwrap() - } - - fn round_one_candidate( - author: AuthorityIndex, - committee: &RbcDagCommitteeContextV1, - marker: u8, - ) -> CandidateCarrierV1 { - let weak_parents = committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(carrier_genesis_reference) - .collect(); - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 1, - own_prev: carrier_genesis_reference(author), - weak_parents, - transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: u64::from(marker), - }, - committee, - ) - .unwrap() - } - - #[test] - fn comparison_backlog_excludes_recovery_and_ages_old_holes() { - let recovered = ShadowDeliverySlotV1 { - author: 0, - round: 10, - }; - let mut backlog = ShadowComparisonBacklogV1::new(BTreeSet::from([recovered])); - assert_eq!(backlog.counts(), (0, 0, 0)); - - // A direct replay can pair with recovered shadow state without - // inventing a current-epoch shadow observation. - backlog.observe_direct(recovered); - assert_eq!(backlog.counts(), (0, 0, 0)); - - let old_hole = ShadowDeliverySlotV1 { - author: 1, - round: 11, - }; - backlog.observe_direct(old_hole); - assert_eq!(backlog.counts(), (1, 0, 0)); - - let newer = ShadowDeliverySlotV1 { - author: 2, - round: 15, - }; - backlog.observe_direct(newer); - backlog.observe_epoch_shadow(newer); - assert_eq!(backlog.counts(), (1, 0, 4)); - - backlog.observe_epoch_shadow(old_hole); - assert_eq!(backlog.counts(), (0, 0, 0)); - } - - #[tokio::test] - async fn service_emits_initial_and_direct_backlog() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start(0, Vec::new()); - wait_ready(&mut events).await; - wait_backlog(&mut events, (0, 0, 0)).await; - - handle - .direct_delivered(ShadowDeliveryIdentityV1::new( - 1, - 7, - TransactionsCommitment::from_bytes([0xB7; 32]), - )) - .unwrap(); - wait_backlog(&mut events, (1, 0, 0)).await; - - stop(handle, events, task).await; - } - - fn phase_carrier( - author: AuthorityIndex, - statement: RbcPhaseStatementV1, - committee: &RbcDagCommitteeContextV1, - ) -> CandidateCarrierV1 { - let previous = |authority: AuthorityIndex| BlockReference { - authority, - round: 1, - digest: BlockDigest::from([0xA0 + authority as u8; 32]), - }; - CandidateCarrierV1::try_new_with_committee( - CarrierHeaderV1Args { - author, - carrier_round: 2, - own_prev: previous(author), - weak_parents: committee - .committee() - .authorities() - .filter(|authority| *authority != author) - .take(2) - .map(previous) - .collect(), - transactions_commitment: TransactionsCommitment::from_bytes( - [0xB0 + author as u8; 32], - ), - data_acknowledgments: Vec::new(), - phase_batch: vec![statement], - consensus_vertex: None, - creation_time_ns: 2, - }, - committee, - ) - .unwrap() - } - - #[tokio::test] - async fn restart_replays_matching_and_rejects_inconsistent_direct_history() { - let harness = Harness::new(); - let direct = direct_header(0, 1, 0x41); - let (handle, mut events, task) = harness.start(0, vec![direct.clone()]); - wait_ready(&mut events).await; - let original = next_carrier(&mut events, 1).await; - stop(handle, events, task).await; - - let (restarted, mut restarted_events, restarted_task) = - harness.start(0, vec![direct.clone()]); - let mut replayed = false; - loop { - match next_event(&mut restarted_events).await { - ShadowServiceEventV1::Recovered { batches, .. } => replayed = batches > 0, - ShadowServiceEventV1::Ready => break, - ShadowServiceEventV1::Rejected { error, .. } => { - panic!("valid shadow restart failed: {error}") - } - _ => {} - } - } - assert!(replayed, "valid shadow restart did not replay its WAL"); - restarted.peer_connected(1).unwrap(); - assert_eq!(next_carrier(&mut restarted_events, 1).await, original); - stop(restarted, restarted_events, restarted_task).await; - - let (missing_handle, missing_events, missing_task) = harness.start(0, Vec::new()); - let missing = startup_rejection(missing_events).await; - assert!(missing.contains("has no matching recovered direct header")); - drop(missing_handle); - missing_task.await.unwrap(); - - let divergent = direct_header(0, 1, 0x42); - let (divergent_handle, divergent_events, divergent_task) = - harness.start(0, vec![divergent]); - let mismatch = startup_rejection(divergent_events).await; - assert!(mismatch.contains("disagree at round 1")); - drop(divergent_handle); - divergent_task.await.unwrap(); - } - - #[tokio::test] - async fn local_carrier_broadcasts_one_identical_full_mac_vector_per_peer() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start(0, Vec::new()); - wait_ready(&mut events).await; - handle.local_header(&direct_header(0, 1, 0x11)).unwrap(); - - let mut by_recipient = BTreeMap::new(); - while by_recipient.len() < N - 1 { - if let ShadowServiceEventV1::Network { - recipient, - message: NetworkMessage::RbcDagShadowCarrier(envelope), - } = next_event(&mut events).await - { - by_recipient.insert(recipient, envelope); - } - } - assert_eq!( - by_recipient.keys().copied().collect::>(), - vec![1, 2, 3] - ); - let first = &by_recipient[&1]; - assert_eq!(first.authentication_sidecar.len(), 3 + N * MAC_TAG_SIZE); - for envelope in by_recipient.values() { - assert_eq!(envelope, first); - } - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &first.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - assert_eq!(candidate.header().carrier_round(), 1); - stop(handle, events, task).await; - } - - #[tokio::test] - async fn poisoned_receiver_mac_is_retained_candidate_only_and_served() { - let harness = Harness::new(); - let (author, mut author_events, author_task) = harness.start(0, Vec::new()); - wait_ready(&mut author_events).await; - author.local_header(&direct_header(0, 1, 0x21)).unwrap(); - let mut envelope = next_carrier(&mut author_events, 2).await; - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &envelope.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - envelope.authentication_sidecar[3 + 2 * MAC_TAG_SIZE] ^= 1; - - let (receiver, mut receiver_events, receiver_task) = harness.start(2, Vec::new()); - wait_ready(&mut receiver_events).await; - receiver.carrier(0, envelope.clone()).unwrap(); - let mut retained = false; - let mut rejected = false; - while !retained || !rejected { - match next_event(&mut receiver_events).await { - ShadowServiceEventV1::Input { - kind: "carrier", - outcome: "retained_unauthenticated", - } => retained = true, - ShadowServiceEventV1::Rejected { peer: Some(0), .. } => rejected = true, - ShadowServiceEventV1::Delivered(_) => { - panic!("candidate-only retention must not grant admission/delivery") - } - _ => {} - } - } - receiver.carrier_request(1, candidate.reference()).unwrap(); - loop { - if let ShadowServiceEventV1::Network { - recipient: 1, - message: NetworkMessage::RbcDagShadowCarrierResponse(response), - } = next_event(&mut receiver_events).await - { - assert_eq!(response.reference, candidate.reference()); - assert_eq!(response.canonical_carrier, envelope.canonical_carrier); - break; - } - } - stop(author, author_events, author_task).await; - stop(receiver, receiver_events, receiver_task).await; - } - - #[tokio::test] - async fn reconnect_replays_exact_persisted_envelope() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start(0, Vec::new()); - wait_ready(&mut events).await; - handle.local_header(&direct_header(0, 1, 0x31)).unwrap(); - let original = next_carrier(&mut events, 1).await; - handle.peer_connected(1).unwrap(); - let first_replay = next_carrier(&mut events, 1).await; - assert_eq!(first_replay, original); - handle.peer_disconnected(1).unwrap(); - tokio::time::sleep(Duration::from_millis(20)).await; - handle.peer_connected(1).unwrap(); - let second_replay = next_carrier(&mut events, 1).await; - assert_eq!(second_replay, original); - stop(handle, events, task).await; - } - - #[tokio::test] - async fn queued_round_waits_for_shadow_quorum_then_broadcasts_exact_next_round() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start(0, Vec::new()); - wait_ready(&mut events).await; - let round_two = direct_header(0, 2, 0x42); - handle.local_header(&round_two).unwrap(); - handle.local_header(&direct_header(0, 1, 0x41)).unwrap(); - let round_one_wire = next_carrier(&mut events, 1).await; - let round_one = CandidateCarrierV1::decode_wire_with_committee( - &round_one_wire.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - assert_eq!(round_one.header().carrier_round(), 1); - - for author in [1, 2] { - let candidate = round_one_candidate(author, &harness.committee, 0x50 + author as u8); - handle - .carrier(author, harness.envelope(&candidate, author)) - .unwrap(); - } - let round_two_wire = loop { - let envelope = next_carrier(&mut events, 1).await; - let candidate = CandidateCarrierV1::decode_wire_with_committee( - &envelope.canonical_carrier, - &harness.committee, - None, - ) - .unwrap(); - if candidate.header().carrier_round() == 2 { - break candidate; - } - }; - assert_eq!( - round_two_wire.header().transactions_commitment(), - round_two.transactions_commitment() - ); - stop(handle, events, task).await; - } - - #[tokio::test] - async fn recovery_binds_holder_and_reference_then_compares_only_paired_slot_once() { - let harness = Harness::new(); - let (handle, mut events, task) = harness.start(3, Vec::new()); - wait_ready(&mut events).await; - handle.peer_connected(0).unwrap(); - handle.peer_connected(1).unwrap(); - let target = round_one_candidate(2, &harness.committee, 0x61); - for sender in [0, 1] { - let outer = phase_carrier( - sender, - RbcPhaseStatementV1::Ready { - target: target.reference(), - }, - &harness.committee, - ); - handle - .carrier(sender, harness.envelope(&outer, sender)) - .unwrap(); - } - let requested = loop { - if let ShadowServiceEventV1::Network { - recipient, - message: NetworkMessage::RbcDagShadowCarrierRequest(reference), - } = next_event(&mut events).await - { - if recipient == 0 || recipient == 1 { - break reference; - } - } - }; - assert_eq!(requested, target.reference()); - - handle - .carrier_response( - 2, - RbcDagShadowCarrierResponse { - reference: target.reference(), - canonical_carrier: target.canonical_wire_bytes().unwrap(), - }, - ) - .unwrap(); - loop { - if let ShadowServiceEventV1::Rejected { - peer: Some(2), - error, - } = next_event(&mut events).await - { - assert!(error.contains("non-holder")); - break; - } - } - - let wrong = round_one_candidate(2, &harness.committee, 0x62); - handle - .carrier_response( - 0, - RbcDagShadowCarrierResponse { - reference: target.reference(), - canonical_carrier: wrong.canonical_wire_bytes().unwrap(), - }, - ) - .unwrap(); - loop { - if let ShadowServiceEventV1::Rejected { - peer: Some(0), - error, - } = next_event(&mut events).await - { - assert!(error.contains("ReferenceMismatch")); - break; - } - } - - let retried = timeout(Duration::from_secs(2), async { - loop { - if let ShadowServiceEventV1::Network { - message: NetworkMessage::RbcDagShadowCarrierRequest(reference), - .. - } = next_event(&mut events).await - { - break reference; - } - } - }) - .await - .unwrap(); - assert_eq!(retried, target.reference()); - - handle - .carrier_response( - 1, - RbcDagShadowCarrierResponse { - reference: target.reference(), - canonical_carrier: target.canonical_wire_bytes().unwrap(), - }, - ) - .unwrap(); - let identity = loop { - if let ShadowServiceEventV1::Delivered(identity) = next_event(&mut events).await { - break identity; - } - }; - assert_eq!(identity.author, 2); - assert_eq!(identity.round, 1); - handle.direct_delivered(identity).unwrap(); - loop { - if let ShadowServiceEventV1::Comparison(comparison) = next_event(&mut events).await { - assert_eq!(comparison, ShadowDeliveryComparisonV1::Match); - break; - } - } - let conflicting = ShadowDeliveryIdentityV1::new( - identity.author, - identity.round, - TransactionsCommitment::from_bytes([0xEE; 32]), - ); - handle.direct_delivered(conflicting).unwrap(); - loop { - if let ShadowServiceEventV1::Comparison(comparison) = next_event(&mut events).await { - assert_eq!( - comparison, - ShadowDeliveryComparisonV1::Ambiguous { - slots: vec![ShadowDeliverySlotV1 { - author: identity.author, - round: identity.round, - }], - } - ); - break; - } - } - stop(handle, events, task).await; - } - - #[tokio::test] - async fn bounded_input_reports_overload_but_shutdown_waits_for_capacity() { - let input_capacity = shadow_input_capacity(N).unwrap(); - let (sender, mut receiver) = mpsc::channel(input_capacity); - let handle = StarfishRbcDagShadowServiceHandleV1 { - sender, - max_sidecar_size: 3 + N * MAC_TAG_SIZE, - own_authority: 0, - committee_size: N, - input_capacity, - desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - invalidated_by_overload: Arc::new(Mutex::new(None)), - }; - for round in 0..input_capacity { - handle - .carrier( - 1, - RbcDagShadowCarrier { - canonical_carrier: vec![round as u8], - authentication_sidecar: Vec::new(), - }, - ) - .unwrap(); - } - assert!(matches!( - handle.carrier( - 1, - RbcDagShadowCarrier { - canonical_carrier: vec![0xFF], - authentication_sidecar: Vec::new(), - }, - ), - Err(ShadowServiceErrorV1::Overloaded { - kind: "carrier", - capacity, - }) - if capacity == input_capacity - )); - - let shutdown_handle = handle.clone(); - let shutdown = tokio::spawn(async move { shutdown_handle.shutdown().await }); - tokio::task::yield_now().await; - assert!(!shutdown.is_finished()); - while let Some(message) = receiver.recv().await { - if let ShadowServiceMessageV1::Shutdown(reply) = message { - reply.send(Ok(())).unwrap(); - break; - } - } - shutdown.await.unwrap().unwrap(); - - let (sender, _receiver) = mpsc::channel(1); - let oversized = StarfishRbcDagShadowServiceHandleV1 { - sender, - max_sidecar_size: 3 + N * MAC_TAG_SIZE, - own_authority: 0, - committee_size: N, - input_capacity: 1, - desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - invalidated_by_overload: Arc::new(Mutex::new(None)), - }; - assert!(matches!( - oversized.carrier( - 1, - RbcDagShadowCarrier { - canonical_carrier: vec![0; MAX_CARRIER_CONTENT_SIZE_V1 + 1], - authentication_sidecar: Vec::new(), - }, - ), - Err(ShadowServiceErrorV1::InputTooLarge { - field: "carrier", - .. - }) - )); - } - - #[tokio::test] - async fn sixty_validator_burst_fits_before_the_actor_drains() { - const LARGE_N: usize = 60; - let input_capacity = shadow_input_capacity(LARGE_N).unwrap(); - assert_eq!(input_capacity, SHADOW_SERVICE_MAX_INPUT_CAPACITY_V1); - let (sender, _receiver) = mpsc::channel(input_capacity); - let invalidated = Arc::new(Mutex::new(None)); - let handle = StarfishRbcDagShadowServiceHandleV1 { - sender, - max_sidecar_size: 3 + LARGE_N * MAC_TAG_SIZE, - own_authority: 0, - committee_size: LARGE_N, - input_capacity, - desired_topology: Arc::new(Mutex::new(BTreeMap::new())), - desired_direct_deliveries: Arc::new(Mutex::new(BTreeSet::new())), - invalidated_by_overload: Arc::clone(&invalidated), - }; - for peer in 1..LARGE_N { - handle - .carrier( - peer as AuthorityIndex, - RbcDagShadowCarrier { - canonical_carrier: vec![peer as u8], - authentication_sidecar: Vec::new(), - }, - ) - .unwrap(); - } - for _ in 0..SHADOW_SERVICE_CONTROL_RESERVE_V1 { - handle.send(ShadowServiceMessageV1::RetryRecovery).unwrap(); - } - assert_eq!(*invalidated.lock(), None); - assert!(matches!( - shadow_input_capacity(LARGE_N + 1), - Err(ShadowServiceErrorV1::CommitteeBurstTooLarge { .. }) - )); - } - - #[tokio::test] - async fn dropping_all_handles_stops_actor_despite_retry_timer() { - let harness = Harness::new(); - let (handle, events, task) = harness.start(0, Vec::new()); - drop(handle); - drop(events); - timeout(EVENT_TIMEOUT, task) - .await - .expect("retry timer retained a strong input sender") - .unwrap(); - } -} diff --git a/crates/starfish-core/src/syncer.rs b/crates/starfish-core/src/syncer.rs index 9f5a68d0..c5b4b422 100644 --- a/crates/starfish-core/src/syncer.rs +++ b/crates/starfish-core/src/syncer.rs @@ -24,7 +24,6 @@ use crate::{ runtime::timestamp_utc, sailfish_service::SailfishServiceMessage, starfish_rbc::{PinnedRbcHeader, RbcCanonicalHeader}, - starfish_rbc_dag_shadow_service::StarfishRbcDagShadowServiceHandleV1, starfish_rbc_service::{RbcLocalHeader, RbcServiceHandle}, types::{ AuthorityIndex, BlockReference, PartialSig, PartialSigKind, ProvableShard, @@ -80,7 +79,6 @@ pub struct Syncer { bls_tx: Option>, sailfish_tx: Option>, starfish_rbc_service: Option, - starfish_rbc_dag_shadow_service: Option, } pub trait SyncerSignals: Send + Sync { @@ -113,7 +111,6 @@ impl Syncer { bls_tx: Option>, sailfish_tx: Option>, starfish_rbc_service: Option, - starfish_rbc_dag_shadow_service: Option, ) -> Self { let committee_size = core.committee().len(); let own_stake = core @@ -135,7 +132,6 @@ impl Syncer { bls_tx, sailfish_tx, starfish_rbc_service, - starfish_rbc_dag_shadow_service, } } @@ -401,18 +397,6 @@ impl Syncer { *block.reference(), "RBC service selected a different local header reference" ); - if let Some(ref shadow) = self.starfish_rbc_dag_shadow_service { - if let Err(error) = shadow.local_header(&canonical) { - self.metrics.starfish_rbc_dag_shadow_comparison_valid.set(0); - self.metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["local", "dropped"]) - .inc(); - tracing::warn!( - "Failed to enqueue non-authoritative RBC-DAG shadow carrier; comparison is invalid: {error}" - ); - } - } } if let Some(started_at) = self.proposal_wait_started_at.take() { self.metrics @@ -692,16 +676,7 @@ mod tests { assert_eq!(core.dag_state().proposal_round(), 3); assert_eq!(core.last_proposed(), 0); - let mut syncer = Syncer::new( - core, - false, - NoopCommitObserver, - metrics, - None, - None, - None, - None, - ); + let mut syncer = Syncer::new(core, false, NoopCommitObserver, metrics, None, None, None); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); syncer.recompute_subscriber_stake(); @@ -799,7 +774,6 @@ mod tests { None, None, None, - None, ); syncer.connected_authorities.extend([1, 2, 3]); syncer.subscribed_by_authorities.extend([1, 2, 3]); diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 7564e7c5..7cc98c6b 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -74,11 +74,6 @@ impl Validator { benchmarks" )); } - if public_config.parameters.starfish_rbc_dag_shadow && !is_starfish_rbc { - return Err(eyre!( - "Starfish-RBC-DAG shadow mode requires consensus 'starfish-rbc'" - )); - } if is_starfish_rbc { let protocol_instance = public_config .parameters @@ -222,7 +217,6 @@ impl Validator { } else { None }; - let starfish_rbc_dag_shadow_wal = private_config.starfish_rbc_dag_shadow_wal(); let (core, bls_cert_aggregator) = Core::open( block_handler, @@ -254,7 +248,6 @@ impl Validator { commit_handler, metrics.clone(), public_config.parameters.clone(), - starfish_rbc_dag_shadow_wal, partial_sig_rx, bls_cert_aggregator, bls_signer_for_service, @@ -317,10 +310,6 @@ mod smoke_tests { use crate::{ committee::Committee, config::{self, NodePrivateConfig, NodePublicConfig, Parameters}, - metrics::{ - STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR, - STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG, - }, prometheus, types::AuthorityIndex, }; @@ -348,54 +337,16 @@ mod smoke_tests { } } - #[tokio::test] - async fn starfish_rbc_dag_shadow_rejects_non_rbc_protocol() { - let committee_size = 4; - let committee = Committee::new_for_benchmarks(committee_size); - let mut public_config = NodePublicConfig::new_for_tests(committee_size); - public_config.parameters.starfish_rbc_dag_shadow = true; - let private_config = - NodePrivateConfig::new_for_benchmarks(TempDir::new().unwrap().as_ref(), committee_size) - .remove(0); - - let result = Validator::start( - 0, - committee, - public_config, - private_config, - Parameters::default(), - "honest".to_string(), - "starfish".to_string(), - ) - .await; - - assert!(result.is_err_and(|error| { - error - .to_string() - .contains("shadow mode requires consensus 'starfish-rbc'") - })); - } - async fn run_commit_test( consensus: &str, block_authentication: Option<&str>, port_offset: u16, - ) { - run_commit_test_with_shadow(consensus, block_authentication, port_offset, false).await; - } - - async fn run_commit_test_with_shadow( - consensus: &str, - block_authentication: Option<&str>, - port_offset: u16, - starfish_rbc_dag_shadow: bool, ) { let committee_size = 4; let committee = Committee::new_for_benchmarks(committee_size); let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); - public_config.parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; if consensus == "starfish-rbc" { public_config .parameters @@ -443,94 +394,6 @@ mod smoke_tests { ), } - if starfish_rbc_dag_shadow { - let maximum_unpaired = STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_FACTOR - * i64::try_from(committee_size).unwrap(); - tokio::time::timeout(timeout, async { - loop { - let complete = validators.iter().all(|validator| { - let metrics = validator.metrics(); - let direct_deliveries = metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "direct"]) - .get(); - let shadow_deliveries = metrics - .starfish_rbc_dag_shadow_inputs_total - .with_label_values(&["delivery", "shadow"]) - .get(); - let matches = metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["match"]) - .get(); - metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1 - && metrics - .starfish_rbc_dag_shadow_wal_durable_records_total - .get() - > 0 - && direct_deliveries > 0 - && shadow_deliveries > 0 - && matches > 0 - && metrics.starfish_rbc_dag_shadow_pending_recovery.get() == 0 - && metrics.starfish_rbc_dag_shadow_unpaired_direct.get() - <= maximum_unpaired - && metrics.starfish_rbc_dag_shadow_unpaired_shadow.get() - <= maximum_unpaired - && metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get() - <= STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG - }); - if complete { - break; - } - time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .expect("shadow did not durably deliver and match direct RBC before timeout"); - - for validator in &validators { - let metrics = validator.metrics(); - assert_eq!( - metrics.starfish_rbc_dag_shadow_comparison_valid.get(), - 1, - "shadow observation stream shed work while direct RBC committed" - ); - assert_eq!( - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["mismatch"]) - .get(), - 0 - ); - assert_eq!( - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["direct_only"]) - .get(), - 0 - ); - assert_eq!( - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["shadow_only"]) - .get(), - 0 - ); - assert_eq!( - metrics - .starfish_rbc_dag_shadow_delivery_comparisons_total - .with_label_values(&["ambiguous"]) - .get(), - 0 - ); - assert!(metrics.starfish_rbc_dag_shadow_unpaired_direct.get() <= maximum_unpaired); - assert!(metrics.starfish_rbc_dag_shadow_unpaired_shadow.get() <= maximum_unpaired); - assert!( - metrics.starfish_rbc_dag_shadow_unpaired_max_round_lag.get() - <= STARFISH_RBC_DAG_SHADOW_MAX_UNPAIRED_ROUND_LAG - ); - } - } - for v in validators { v.stop().await; } @@ -580,11 +443,6 @@ mod smoke_tests { run_commit_test("bluestreak", None, 150).await; } - #[tokio::test] - async fn starfish_rbc_dag_shadow_mac_keeps_direct_commits_live() { - run_commit_test_with_shadow("starfish-rbc", Some("mac"), 1640, true).await; - } - #[tokio::test] async fn starfish_rbc_single_validator_starts_on_current_thread_runtime() { let committee_size = 4; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index 6a448240..bdafb49d 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -72,10 +72,6 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Run the persisted, non-authoritative RBC-DAG shadow alongside - /// `starfish-rbc`. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_shadow: bool, }, /// Deploy a local validator for test. Dryrun mode uses /// default keys and committee configurations. @@ -109,10 +105,6 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Run the persisted, non-authoritative RBC-DAG shadow alongside - /// `starfish-rbc`. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_shadow: bool, /// Directory to store validator data (default: current directory) #[clap(long, value_name = "PATH")] data_dir: Option, @@ -168,10 +160,6 @@ enum Operation { /// to the experimental `*-mac` protocols. #[clap(long, value_name = "ed25519|ml-dsa-44|ml-dsa-65|mac")] block_authentication: Option, - /// Run the persisted, non-authoritative RBC-DAG shadow alongside - /// `starfish-rbc`. - #[clap(long, default_value_t = false)] - starfish_rbc_dag_shadow: bool, /// Testbed-only: deliver a single-DAG RBC header after a receiver-local /// quorum ECHO. This preserves uniqueness but not Byzantine /// selective-withholding totality, so it is restricted to finite @@ -212,7 +200,6 @@ async fn main() -> Result<()> { byzantine_strategy, consensus: consensus_protocol, block_authentication, - starfish_rbc_dag_shadow, } => { run( authority, @@ -223,7 +210,6 @@ async fn main() -> Result<()> { byzantine_strategy, consensus_protocol, block_authentication, - starfish_rbc_dag_shadow, ) .await? } @@ -238,7 +224,6 @@ async fn main() -> Result<()> { adversarial_latency_percent, consensus: consensus_protocol, block_authentication, - starfish_rbc_dag_shadow, data_dir, base_ip, storage_backend, @@ -258,7 +243,6 @@ async fn main() -> Result<()> { adversarial_latency_percent, consensus_protocol, block_authentication, - starfish_rbc_dag_shadow, data_dir, base_ip, storage_backend, @@ -280,7 +264,6 @@ async fn main() -> Result<()> { adversarial_latency_percent, consensus: consensus_protocol, block_authentication, - starfish_rbc_dag_shadow, starfish_rbc_single_dag_echo_qc_fast_path, duration_secs, dissemination_mode, @@ -292,7 +275,6 @@ async fn main() -> Result<()> { node_parameters.adversarial_latency = adversarial_latency; node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.block_authentication = block_authentication; - node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; node_parameters.starfish_rbc_single_dag_echo_qc_fast_path = starfish_rbc_single_dag_echo_qc_fast_path; if is_starfish_rbc_selection(&consensus_protocol) { @@ -444,7 +426,6 @@ async fn local_benchmark( parameters.clone() }; let public_config = NodePublicConfig::new_for_benchmarks(ips, Some(node_parameters.clone())); - let starfish_rbc_dag_shadow_expected = node_parameters.starfish_rbc_dag_shadow; // Create temporary directories for each validator let base_dir = PathBuf::from("local-benchmark"); @@ -531,30 +512,6 @@ async fn local_benchmark( handles.push(handle); } - if starfish_rbc_dag_shadow_expected { - let ready = tokio::time::timeout(Duration::from_secs(30), async { - loop { - if metrics_of_honest_validators - .iter() - .all(|metrics| metrics.starfish_rbc_dag_shadow_comparison_valid.get() == 1) - { - break; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await; - if ready.is_err() { - for abort_handle in &abort_handles { - abort_handle.abort(); - } - fs::remove_dir_all(&base_dir)?; - eyre::bail!( - "Starfish-RBC-DAG shadow did not become ready on every honest validator; benchmark was not started" - ); - } - } - // Run for specified duration tokio::select! { _ = tokio::time::sleep(Duration::from_secs(duration_secs)) => { @@ -567,8 +524,6 @@ async fn local_benchmark( metrics_of_honest_validators, reporters_of_honest_validators, duration_secs, - committee_size, - starfish_rbc_dag_shadow_expected, ); // Abort all tasks @@ -592,8 +547,6 @@ async fn local_benchmark( metrics_of_honest_validators, reporters_of_honest_validators, duration_secs, - committee_size, - starfish_rbc_dag_shadow_expected, ); fs::remove_dir_all(base_dir)?; Ok(()) @@ -611,7 +564,6 @@ async fn run( byzantine_strategy: String, consensus_protocol: String, block_authentication: Option, - starfish_rbc_dag_shadow: bool, ) -> Result<()> { tracing::info!("Starting node {authority}"); @@ -623,9 +575,6 @@ async fn run( if block_authentication.is_some() { public_config.parameters.block_authentication = block_authentication; } - if starfish_rbc_dag_shadow { - public_config.parameters.starfish_rbc_dag_shadow = true; - } let private_config = NodePrivateConfig::load(&private_config_path).wrap_err(format!( "Failed to load private configuration file '{private_config_path}'" ))?; @@ -663,7 +612,6 @@ async fn dryrun( adversarial_latency_percent: u32, consensus_protocol: String, block_authentication: Option, - starfish_rbc_dag_shadow: bool, data_dir: Option, base_ip: Option, storage_backend: Option, @@ -706,8 +654,6 @@ async fn dryrun( node_parameters.adversarial_latency_percent = adversarial_latency_percent; node_parameters.compress_network = compress_network; node_parameters.block_authentication = block_authentication; - node_parameters.starfish_rbc_dag_shadow = starfish_rbc_dag_shadow; - ensure_starfish_rbc_protocol_instance(&consensus_protocol, &mut node_parameters); if let Some(workers) = bls_workers { node_parameters.bls_verification_workers = workers; } @@ -887,8 +833,7 @@ mod tests { use clap::Parser; - use super::{Args, Operation, ensure_starfish_rbc_protocol_instance, ipv4_add_offset}; - use starfish_core::config::NodeParameters; + use super::{Args, Operation, ipv4_add_offset}; #[test] fn ipv4_add_offset_crosses_octet_boundary() { @@ -944,7 +889,6 @@ mod tests { "starfish-rbc", "--block-authentication", "mac", - "--starfish-rbc-dag-shadow", "--starfish-rbc-single-dag-echo-qc-fast-path", ]) .unwrap(); @@ -952,7 +896,6 @@ mod tests { let Operation::LocalBenchmark { consensus, block_authentication, - starfish_rbc_dag_shadow, starfish_rbc_single_dag_echo_qc_fast_path, .. } = args.operation @@ -961,7 +904,6 @@ mod tests { }; assert_eq!(consensus, "starfish-rbc"); assert_eq!(block_authentication.as_deref(), Some("mac")); - assert!(starfish_rbc_dag_shadow); assert!(starfish_rbc_single_dag_echo_qc_fast_path); } @@ -979,6 +921,5 @@ mod tests { .starfish_rbc_protocol_instance .is_some_and(|instance| instance != [0; 32]) ); - assert!(parameters.starfish_rbc_dag_shadow); } } diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 9753358d..385a9c54 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -10,11 +10,8 @@ Status: standalone MAC-vector RBC-DAG prototype with authoritative optimistic de committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoint transfer, and full validator recovery remain incomplete -The provisional CLI name for the eventual protocol is `starfish-rbc-dag`. That selector is not -implemented. The current runtime is enabled with `--consensus starfish-rbc ---starfish-rbc-dag-shadow`; it observes the direct prototype without changing its DAG, pacemaker, -commit, or output. The eventual protocol is new, not a transport option or a version-two alias for -`starfish-rbc`. +The provisional CLI name for this protocol is `starfish-rbc-dag`. It is a new protocol, not a +transport option or a version-two alias for `starfish-rbc`. The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through @@ -44,43 +41,13 @@ remain selectable outer-authentication baselines; changing that selector does no embedded RBC or consensus rules. This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish -commit rules already exist. Milestone two provides a canonical codec plus deterministic -carrier/RBC, certified-projection, decision, and crash-journal models. Milestone three adds an -opt-in persisted shadow actor, full-vector carrier transport, recovery messages, and paired -direct/shadow delivery observations. The direct `starfish-rbc` service remains the only authority: -shadow admission, delivery, recovery, or failure cannot advance a proposal, mark a DAG vertex -clean, vote, commit, or order output. - -Milestone-three restart coverage is deliberately scoped to reopening the shadow actor and its WAL -against an identical recovered direct-header history. It is not a full validator crash-recovery -claim. The authoritative direct `starfish-rbc` baseline does not yet durably record its remote-slot -ECHO/READY choices, delivery locks, or retained phase evidence. Restarting that baseline after it -has proposed a non-genesis block can therefore forget proof-critical choices and leave the newest -recovered own header dirty. Full validator restart remains fail-stop until those direct-RBC locks -are persisted and replayed; replaying only the observational shadow WAL cannot repair or safely -substitute for them. - -That isolation is logical, not physical: shadow frames share the validator's existing TCP -connections, outbound queues, bandwidth, and CPU process with direct RBC, so enabling the shadow -can perturb authoritative timing even though no shadow result is consumed by consensus. Version -one has no feature handshake. Every validator in a shadow run must use a binary that understands -the append-only shadow wire variants and the flag must be deployed committee-wide; an older peer -will reject an unknown bincode variant and may close the shared connection. Mixed-version or -partially enabled runs are not valid comparisons. - -Shadow shutdown is bounded so an observational WAL failure cannot indefinitely block validator -shutdown. If that timeout fires, the blocking worker may still hold the shadow WAL's single-writer -handle even after its async supervisor is detached. A same-process restart against that storage is -therefore forbidden until the worker has exited; process exit remains safe. A production-quality -same-process restart path needs an operating-system file lock or a fully cancellable storage task. - -The shadow runtime is not a proof or a performance implementation. It intentionally fsyncs every -accepted transition and its reference reducer clones retained model/journal history, so total CPU -work grows superlinearly with a long run. It also uses a fixed unsolicited-retention window only as -a benchmark resource guard; that window is not a safe asynchronous pruning rule. Until the -composition, resource bounds, and performance path are completed, `starfish-rbc-dag` must be -described as an experimental shadow/reference implementation rather than a proven signature-free -Starfish variant or a fair throughput baseline. +commit rules already exist. The isolated milestone-two implementation now provides a canonical +codec plus deterministic carrier/RBC, certified-projection, decision, and crash-journal models. +Those models are not runtime integration or a proof: the composition through two clocks, two +logical projections, and frontier-based payload ordering still requires shadow execution, +additional adversarial testing, and a safety/liveness argument. Until those obligations are +discharged, `starfish-rbc-dag` must be described as an experimental reference model rather than a +proven signature-free Starfish variant. The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not @@ -742,40 +709,19 @@ Hash-sorting recovered carriers is not a valid reconstruction rule. Byzantine eq arrival order determine which value a local slot-global guard selects, and a different restart order could make one honest authority appear to send conflicting phases. -The proof model retains all proof-critical carrier, phase, prefix, and consensus state for the run. -The milestone-three shadow bounds newly arriving unsolicited content to a fixed recent-round -window solely to keep a faulty peer from growing an observational benchmark process without limit. -This is not a protocol-safe retirement rule: an honest INIT may be delayed longer than that under -asynchrony. Recovery of an exact already-requested value is exempt. Before authoritative garbage -collection is enabled, the design needs a common certified or committed retirement watermark that -preserves: +The initial model and shadow prototype retain all proof-critical carrier, phase, prefix, and +consensus state for the run. Before garbage collection is enabled, the design needs a common +retirement watermark that preserves: - pending Bracha totality and header recovery; - exact self-prefix expansion from the last committed frontier; - committed-anchor reconstruction for a late validator; and - deterministic replay of local locks. -Resource bounds still required before authoritative deployment include a proof-safe future and -retirement window, per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, -a bounded payload runahead policy, and checkpointed disk-backed recovery. Shadow input and output -channels are bounded and shed observational work instead of backpressuring direct consensus, but -the reference reducer's retained history and per-transition validation are not yet bounded-runtime -architecture. Resource exhaustion is excluded from the initial proof model and must be measured in -the prototype. Any run in which work is shed is invalid for direct/shadow comparison; -`starfish_rbc_dag_shadow_comparison_valid` must remain `1` for the entire measured interval. A live -pipeline does not have equal cumulative direct and shadow delivery counters at an arbitrary -instant: embedded ECHO/READY normally leaves a short shadow tail. Benchmark verification therefore -requires monotone nonzero direct, shadow, and paired-match progress, no conflict outcome, and bounds -both the current unpaired slots (`<= 4n` per validator) and the oldest unpaired round lag against the -newest current-process observation (`<= 4`). These are empirical benchmark coverage guards, not -asynchronous protocol bounds; a run exceeding either guard is discarded rather than treated as -proof of a protocol failure. - -The milestone-three actor reserves the full hard 64-entry queue so several fan-in bursts can wait -behind a synchronous fsync, capping queued maximum-sized carrier bodies at 256 MiB (plus sidecars -and allocator overhead). It still verifies that one peer fan-in plus five local/control inputs fits; -shadow runs above 60 validators are rejected rather than silently producing an incomplete -comparison. +Resource bounds still required before authoritative deployment include a future carrier window, +per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, a bounded payload +runahead policy, and disk-backed recovery. Resource exhaustion is excluded from the initial proof +model but must be measured in the prototype. ## 15. Safety obligations @@ -845,10 +791,8 @@ minimum it must cover: - equal committed anchors producing byte-identical output deltas; - delayed data availability followed by eventual prefix inclusion; - crash points before and after each persisted lock and outbound-carrier write; and -- persisted shadow-actor restart with byte-identical retransmission against an identical recovered - direct-header history, bounded overload, poisoned-tag candidate retention, exact recovery, and - paired delivery observations against the current direct RBC kernel. Full validator restart is - excluded until the authoritative direct-RBC locks are durable. +- once milestone three supplies the non-authoritative runtime path, shadow replay matching the + current direct RBC kernel's delivered references. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -873,10 +817,7 @@ Batching can reduce the number of separately scheduled RBC control messages, but their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout transport is added. Shadow mode also sends both direct and embedded transcripts and is a correctness -instrument, not a performance result. In milestone three it additionally fsyncs each accepted -transition and validates through a clone-based reference reducer. Those costs are deliberately not -charged as protocol overhead: performance runs require incremental state transitions/checkpoints -or an equivalently durable baseline, plus separate WAL/fsync accounting. +instrument, not a performance result. ## 19. Contained implementation milestones @@ -889,12 +830,10 @@ Every milestone is committed separately. frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable journal models; and deterministic adversarial simulations. No network or existing consensus path changes. -3. **Persisted shadow carrier path (implemented, opt-in):** build and store carriers alongside the current direct +3. **Persisted shadow carrier path:** build and store carriers alongside the current direct `starfish-rbc` service, cache the validated committee/domain identity rather than re-hashing all public keys per carrier, journal ingress and local locks, and compare embedded versus direct RBC - delivery through current-process paired observations. Direct RBC remains authoritative; shadow - results never affect proposals or commits. The reference WAL/reducer is a correctness instrument, - not yet an interpretable protocol-performance path. + delivery. Direct RBC remains authoritative; shadow results never affect proposals or commits. 4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses the current baseline. From b4721ad020fb310fcf6b513dc37f78505077f04d Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:46:19 +0200 Subject: [PATCH 60/62] Revert "Add Starfish-RBC-DAG codec and executable model" This reverts commit e29647900d69341c128f0c81aadf439b09e448ca. --- README.md | 11 +- crates/starfish-core/src/crypto.rs | 12 - crates/starfish-core/src/lib.rs | 1 - .../src/starfish_rbc_dag/journal.rs | 2220 --------------- .../starfish-core/src/starfish_rbc_dag/mod.rs | 2516 ----------------- .../src/starfish_rbc_dag/model.rs | 2266 --------------- .../src/starfish_rbc_dag/projection.rs | 1548 ---------- docs/starfish-rbc-dag-protocol.md | 140 +- 8 files changed, 41 insertions(+), 8673 deletions(-) delete mode 100644 crates/starfish-core/src/starfish_rbc_dag/journal.rs delete mode 100644 crates/starfish-core/src/starfish_rbc_dag/mod.rs delete mode 100644 crates/starfish-core/src/starfish_rbc_dag/model.rs delete mode 100644 crates/starfish-core/src/starfish_rbc_dag/projection.rs diff --git a/README.md b/README.md index 3408e20c..7ad5fe03 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,11 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is a codec-and-model-only follow-up that pipelines all-carrier RBC through an -optimistic carrier DAG while keeping certified Starfish consensus and ordering in a separate -logical projection. Its canonical types and deterministic executable models are implemented in -isolation, but there is no network/runtime path and no safety or liveness claim. Its provisional CLI -name is `starfish-rbc-dag`, but that selector is not implemented yet. The design, current boundary, -and proof obligations are documented in the [protocol design](docs/starfish-rbc-dag-protocol.md). +**Starfish-RBC-DAG** is a design-only follow-up that pipelines all-carrier RBC through an optimistic +carrier DAG while keeping certified Starfish consensus and ordering in a separate logical +projection. Its provisional CLI name is `starfish-rbc-dag`, but that selector is not implemented +yet. The full design and proof obligations are documented in the +[protocol design](docs/starfish-rbc-dag-protocol.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/starfish-core/src/crypto.rs b/crates/starfish-core/src/crypto.rs index 3d85e0bf..430ee2a3 100644 --- a/crates/starfish-core/src/crypto.rs +++ b/crates/starfish-core/src/crypto.rs @@ -128,11 +128,6 @@ impl Hasher for Blake3 { } impl TransactionsCommitment { - /// Construct a transaction commitment from its canonical 32-byte form. - pub const fn from_bytes(bytes: [u8; TRANSACTIONS_DIGEST_SIZE]) -> Self { - Self(bytes) - } - pub fn new_from_encoded_transactions( encoded_transactions: &Vec, authority_index: usize, @@ -967,13 +962,6 @@ impl AsRef<[u8]> for SignatureBytes { } } -impl SignatureBytes { - /// Construct an Ed25519 signature from its canonical fixed-width form. - pub const fn from_bytes(bytes: [u8; SIGNATURE_SIZE]) -> Self { - Self(bytes) - } -} - impl AsBytes for TransactionsCommitment { fn as_bytes(&self) -> &[u8] { &self.0 diff --git a/crates/starfish-core/src/lib.rs b/crates/starfish-core/src/lib.rs index 70c7f628..bd6068cc 100644 --- a/crates/starfish-core/src/lib.rs +++ b/crates/starfish-core/src/lib.rs @@ -30,7 +30,6 @@ mod rocks_store; mod runtime; pub mod shard_reconstructor; pub mod starfish_rbc; -pub mod starfish_rbc_dag; mod starfish_rbc_service; mod stat; mod state; diff --git a/crates/starfish-core/src/starfish_rbc_dag/journal.rs b/crates/starfish-core/src/starfish_rbc_dag/journal.rs deleted file mode 100644 index fa7287a6..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag/journal.rs +++ /dev/null @@ -1,2220 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Crash/restart model for proof-critical Starfish-RBC-DAG state. -//! -//! This module is intentionally a deterministic write-ahead-log reducer. It -//! does not perform I/O and it does not duplicate carrier or authentication -//! decoding. Callers validate canonical bytes before journaling them; the -//! reducer pins the exact byte strings and rejects any later alternative. - -use std::{collections::BTreeMap, error::Error, fmt}; - -use crate::types::{AuthorityIndex, BlockReference, RoundNumber}; - -use super::{ - AuthenticatedCarrierV1, CandidateCarrierV1, LeaderChoiceV1, LocallyAuthenticatedCarrierV1, - RbcDagContextV1, RbcPhaseStatementV1, -}; - -/// A Bracha slot is identified independently of the candidate digest. -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct RbcSlotKeyV1 { - pub author: AuthorityIndex, - pub carrier_round: RoundNumber, -} - -impl RbcSlotKeyV1 { - pub const fn new(author: AuthorityIndex, carrier_round: RoundNumber) -> Self { - Self { - author, - carrier_round, - } - } - - pub const fn of(reference: BlockReference) -> Self { - Self::new(reference.authority, reference.round) - } -} - -/// Provenance retained for an authenticated ingress record. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum IngressProvenanceV1 { - DirectFromAuthor, - Relayed { peer: AuthorityIndex }, -} - -/// One authenticated arrival in its locally observed order. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct AuthenticatedIngressRecordV1 { - sequence: u64, - reference: BlockReference, - provenance: IngressProvenanceV1, - canonical_carrier_wire: Vec, - authentication_sidecar: Vec, -} - -impl AuthenticatedIngressRecordV1 { - pub fn sequence(&self) -> u64 { - self.sequence - } - - pub fn reference(&self) -> BlockReference { - self.reference - } - - pub fn provenance(&self) -> IngressProvenanceV1 { - self.provenance - } - - pub fn canonical_carrier_wire(&self) -> &[u8] { - &self.canonical_carrier_wire - } - - pub fn authentication_sidecar(&self) -> &[u8] { - &self.authentication_sidecar - } -} - -/// Exact local carrier bytes retained for first send and retransmission. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct DurableOutboundCarrierV1 { - reference: BlockReference, - canonical_carrier_wire: Vec, - authentication_sidecar: Vec, - exposed: bool, -} - -impl DurableOutboundCarrierV1 { - pub fn reference(&self) -> BlockReference { - self.reference - } - - pub fn canonical_carrier_wire(&self) -> &[u8] { - &self.canonical_carrier_wire - } - - pub fn authentication_sidecar(&self) -> &[u8] { - &self.authentication_sidecar - } - - pub fn exposed(&self) -> bool { - self.exposed - } -} - -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub enum PhaseKindV1 { - Echo, - Ready, -} - -/// Durable result of processing one entry in an enclosing phase batch. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum AppliedPhaseOutcomeV1 { - Counted, - IgnoredReplay, - IgnoredEquivocation, -} - -impl PhaseKindV1 { - const fn of(statement: RbcPhaseStatementV1) -> Self { - match statement { - RbcPhaseStatementV1::Echo { .. } => Self::Echo, - RbcPhaseStatementV1::Ready { .. } => Self::Ready, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -struct SenderPhaseKeyV1 { - slot: RbcSlotKeyV1, - sender: AuthorityIndex, - phase: PhaseKindV1, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct AppliedPhaseEntryV1 { - sender: AuthorityIndex, - statement: RbcPhaseStatementV1, - outcome: AppliedPhaseOutcomeV1, -} - -/// One durable write-ahead event. -/// -/// The context is repeated in every event so that copied records from another -/// protocol instance, committee, or authentication run fail closed on replay. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum JournalEventV1 { - AuthenticatedIngress { - context: RbcDagContextV1, - sequence: u64, - authenticated: AuthenticatedCarrierV1, - provenance: IngressProvenanceV1, - }, - /// Canonical content retained through header recovery. This is content - /// availability for READY/delivery, not authenticated admission for ECHO. - RetainCandidateContent { - context: RbcDagContextV1, - candidate: CandidateCarrierV1, - }, - FixOwnCarrier { - context: RbcDagContextV1, - reference: BlockReference, - }, - LockEcho { - context: RbcDagContextV1, - target: BlockReference, - }, - LockAdmission { - context: RbcDagContextV1, - target: BlockReference, - }, - LockReady { - context: RbcDagContextV1, - target: BlockReference, - }, - LockDelivery { - context: RbcDagContextV1, - target: BlockReference, - }, - LockConsensusSlot { - context: RbcDagContextV1, - consensus_round: RoundNumber, - enclosing_carrier: BlockReference, - }, - LockLeaderChoice { - context: RbcDagContextV1, - consensus_round: RoundNumber, - choice: LeaderChoiceV1, - }, - PersistOutboundContent { - context: RbcDagContextV1, - candidate: CandidateCarrierV1, - }, - PersistOutboundSidecar { - context: RbcDagContextV1, - authenticated: LocallyAuthenticatedCarrierV1, - }, - ExposeOutbound { - context: RbcDagContextV1, - reference: BlockReference, - }, - ApplyPhaseStatement { - context: RbcDagContextV1, - outer: BlockReference, - index: usize, - sender: AuthorityIndex, - statement: RbcPhaseStatementV1, - }, - AdvancePhaseBatchCursor { - context: RbcDagContextV1, - outer: BlockReference, - index: usize, - }, -} - -impl JournalEventV1 { - fn context(&self) -> RbcDagContextV1 { - match self { - Self::AuthenticatedIngress { context, .. } - | Self::RetainCandidateContent { context, .. } - | Self::FixOwnCarrier { context, .. } - | Self::LockEcho { context, .. } - | Self::LockAdmission { context, .. } - | Self::LockReady { context, .. } - | Self::LockDelivery { context, .. } - | Self::LockConsensusSlot { context, .. } - | Self::LockLeaderChoice { context, .. } - | Self::PersistOutboundContent { context, .. } - | Self::PersistOutboundSidecar { context, .. } - | Self::ExposeOutbound { context, .. } - | Self::ApplyPhaseStatement { context, .. } - | Self::AdvancePhaseBatchCursor { context, .. } => *context, - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct PartialOutboundCarrierV1 { - reference: BlockReference, - candidate: CandidateCarrierV1, - canonical_carrier_wire: Vec, - authentication_sidecar: Option>, - exposed: bool, -} - -impl PartialOutboundCarrierV1 { - fn new(candidate: CandidateCarrierV1, canonical_carrier_wire: Vec) -> Self { - Self { - reference: candidate.reference(), - candidate, - canonical_carrier_wire, - authentication_sidecar: None, - exposed: false, - } - } - - fn complete(&self) -> Option { - Some(DurableOutboundCarrierV1 { - reference: self.reference, - canonical_carrier_wire: self.canonical_carrier_wire.clone(), - authentication_sidecar: self.authentication_sidecar.clone()?, - exposed: self.exposed, - }) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct RetainedCarrierV1 { - candidate: CandidateCarrierV1, - canonical_carrier_wire: Vec, -} - -/// State reconstructed exclusively from the durable event sequence. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct JournalSnapshotV1 { - context: RbcDagContextV1, - own_authority: AuthorityIndex, - ingress: Vec, - retained_carriers: BTreeMap, - own_carriers: BTreeMap, - admission_locks: BTreeMap, - echo_locks: BTreeMap, - ready_locks: BTreeMap, - delivery_locks: BTreeMap, - consensus_slots: BTreeMap, - leader_choices: BTreeMap, - outbound: BTreeMap, - applied_phase_entries: BTreeMap<(BlockReference, usize), AppliedPhaseEntryV1>, - sender_phase_locks: BTreeMap, - phase_batch_cursors: BTreeMap, -} - -impl JournalSnapshotV1 { - fn new(context: RbcDagContextV1, own_authority: AuthorityIndex) -> Self { - Self { - context, - own_authority, - ingress: Vec::new(), - retained_carriers: BTreeMap::new(), - own_carriers: BTreeMap::new(), - admission_locks: BTreeMap::new(), - echo_locks: BTreeMap::new(), - ready_locks: BTreeMap::new(), - delivery_locks: BTreeMap::new(), - consensus_slots: BTreeMap::new(), - leader_choices: BTreeMap::new(), - outbound: BTreeMap::new(), - applied_phase_entries: BTreeMap::new(), - sender_phase_locks: BTreeMap::new(), - phase_batch_cursors: BTreeMap::new(), - } - } - - pub fn context(&self) -> RbcDagContextV1 { - self.context - } - - pub fn own_authority(&self) -> AuthorityIndex { - self.own_authority - } - - pub fn authenticated_ingress(&self) -> &[AuthenticatedIngressRecordV1] { - &self.ingress - } - - pub fn next_ingress_sequence(&self) -> u64 { - self.ingress.len() as u64 - } - - pub fn retained_carrier(&self, reference: BlockReference) -> Option<&[u8]> { - self.retained_carriers - .get(&reference) - .map(|retained| retained.canonical_carrier_wire.as_slice()) - } - - pub fn own_carrier(&self, round: RoundNumber) -> Option { - self.own_carriers.get(&round).copied() - } - - pub fn admission_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.admission_locks.get(&slot).copied() - } - - pub fn echo_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.echo_locks.get(&slot).copied() - } - - pub fn ready_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.ready_locks.get(&slot).copied() - } - - pub fn delivery_lock(&self, slot: RbcSlotKeyV1) -> Option { - self.delivery_locks.get(&slot).copied() - } - - pub fn consensus_slot(&self, round: RoundNumber) -> Option { - self.consensus_slots.get(&round).copied() - } - - pub fn leader_choice(&self, round: RoundNumber) -> Option { - self.leader_choices.get(&round).copied() - } - - pub fn phase_batch_cursor(&self, outer: BlockReference) -> usize { - self.phase_batch_cursors.get(&outer).copied().unwrap_or(0) - } - - pub fn phase_statement_applied(&self, outer: BlockReference, index: usize) -> bool { - self.applied_phase_entries.contains_key(&(outer, index)) - } - - pub fn phase_statement_outcome( - &self, - outer: BlockReference, - index: usize, - ) -> Option { - self.applied_phase_entries - .get(&(outer, index)) - .map(|entry| entry.outcome) - } - - pub fn outbound(&self, reference: BlockReference) -> Option { - self.outbound - .get(&reference) - .and_then(PartialOutboundCarrierV1::complete) - } - - /// Byte-identical records that a restart must retransmit. - pub fn retransmissions(&self) -> Vec { - self.outbound - .values() - .filter_map(PartialOutboundCarrierV1::complete) - .filter(DurableOutboundCarrierV1::exposed) - .collect() - } - - fn apply(&mut self, event: &JournalEventV1) -> Result<(), JournalErrorV1> { - if event.context() != self.context { - return Err(JournalErrorV1::ContextMismatch); - } - - match event { - JournalEventV1::AuthenticatedIngress { - sequence, - authenticated, - provenance, - .. - } => self.apply_ingress(*sequence, authenticated, *provenance), - JournalEventV1::RetainCandidateContent { candidate, .. } => { - self.retain_carrier_content(candidate).map(|_| ()) - } - JournalEventV1::FixOwnCarrier { reference, .. } => self.fix_own_carrier(*reference), - JournalEventV1::LockAdmission { target, .. } => self.lock_admission(*target), - JournalEventV1::LockEcho { target, .. } => self.lock_echo(*target), - JournalEventV1::LockReady { target, .. } => self.lock_ready(*target), - JournalEventV1::LockDelivery { target, .. } => { - self.ensure_retained(*target)?; - if self.ready_lock(RbcSlotKeyV1::of(*target)) != Some(*target) { - return Err(JournalErrorV1::DeliveryWithoutMatchingReady(*target)); - } - Self::lock_candidate(&mut self.delivery_locks, *target, LockKindV1::Delivery) - } - JournalEventV1::LockConsensusSlot { - consensus_round, - enclosing_carrier, - .. - } => self.lock_consensus_slot(*consensus_round, *enclosing_carrier), - JournalEventV1::LockLeaderChoice { - consensus_round, - choice, - .. - } => self.lock_leader_choice(*consensus_round, *choice), - JournalEventV1::PersistOutboundContent { candidate, .. } => { - self.persist_outbound_content(candidate) - } - JournalEventV1::PersistOutboundSidecar { authenticated, .. } => { - self.persist_outbound_sidecar(authenticated) - } - JournalEventV1::ExposeOutbound { reference, .. } => self.expose_outbound(*reference), - JournalEventV1::ApplyPhaseStatement { - outer, - index, - sender, - statement, - .. - } => self.apply_phase_statement(*outer, *index, *sender, *statement), - JournalEventV1::AdvancePhaseBatchCursor { outer, index, .. } => { - self.advance_phase_cursor(*outer, *index) - } - } - } - - fn apply_ingress( - &mut self, - sequence: u64, - authenticated: &AuthenticatedCarrierV1, - provenance: IngressProvenanceV1, - ) -> Result<(), JournalErrorV1> { - let expected = self.next_ingress_sequence(); - if sequence != expected { - return Err(JournalErrorV1::IngressSequence { - expected, - actual: sequence, - }); - } - if authenticated.context() != self.context { - return Err(JournalErrorV1::AuthenticatedIngressContextMismatch); - } - if authenticated.receiver() != self.own_authority { - return Err(JournalErrorV1::AuthenticatedIngressReceiverMismatch { - expected: self.own_authority, - actual: authenticated.receiver(), - }); - } - let reference = authenticated.reference(); - let canonical_carrier_wire = self.retain_carrier_content(authenticated.candidate())?; - let authentication_sidecar = authenticated.authentication().canonical_wire_bytes(); - self.ingress.push(AuthenticatedIngressRecordV1 { - sequence, - reference, - provenance, - canonical_carrier_wire, - authentication_sidecar, - }); - Ok(()) - } - - fn fix_own_carrier(&mut self, reference: BlockReference) -> Result<(), JournalErrorV1> { - if reference.authority != self.own_authority { - return Err(JournalErrorV1::OwnCarrierAuthorMismatch { - expected: self.own_authority, - actual: reference.authority, - }); - } - if reference.round == 0 { - return Err(JournalErrorV1::EncodedGenesisCarrier); - } - if !self.outbound.contains_key(&reference) { - return Err(JournalErrorV1::OutboundContentNotPersisted(reference)); - } - Self::lock_exact( - &mut self.own_carriers, - reference.round, - reference, - JournalErrorV1::ConflictingOwnCarrier(reference.round), - ) - } - - fn lock_echo(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - self.ensure_retained(target)?; - let admitted = self.admission_lock(RbcSlotKeyV1::of(target)) == Some(target); - let fixed_locally = self.own_carrier(target.round) == Some(target); - if !admitted && !fixed_locally { - return Err(JournalErrorV1::EchoWithoutAdmission(target)); - } - Self::lock_candidate(&mut self.echo_locks, target, LockKindV1::Echo) - } - - fn lock_admission(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - let authenticated_ingress = self - .ingress - .iter() - .any(|ingress| ingress.reference == target); - if !authenticated_ingress { - return Err(JournalErrorV1::AdmissionWithoutAuthenticatedIngress(target)); - } - Self::lock_candidate(&mut self.admission_locks, target, LockKindV1::Admission) - } - - fn lock_ready(&mut self, target: BlockReference) -> Result<(), JournalErrorV1> { - self.ensure_retained(target)?; - Self::lock_candidate(&mut self.ready_locks, target, LockKindV1::Ready) - } - - fn ensure_retained(&self, reference: BlockReference) -> Result<(), JournalErrorV1> { - if self.retained_carriers.contains_key(&reference) { - Ok(()) - } else { - Err(JournalErrorV1::CarrierContentNotRetained(reference)) - } - } - - fn retain_carrier_content( - &mut self, - candidate: &CandidateCarrierV1, - ) -> Result, JournalErrorV1> { - if candidate.committee_id() != self.context.committee_id() { - return Err(JournalErrorV1::CandidateCommitteeMismatch); - } - let reference = candidate.reference(); - let canonical_carrier_wire = candidate - .canonical_wire_bytes() - .map_err(|_| JournalErrorV1::CanonicalCarrierEncoding(reference))?; - match self.retained_carriers.get(&reference) { - Some(existing) - if existing.candidate != *candidate - || existing.canonical_carrier_wire != canonical_carrier_wire => - { - Err(JournalErrorV1::ConflictingRetainedContent(reference)) - } - Some(existing) => Ok(existing.canonical_carrier_wire.clone()), - None => { - self.retained_carriers.insert( - reference, - RetainedCarrierV1 { - candidate: candidate.clone(), - canonical_carrier_wire: canonical_carrier_wire.clone(), - }, - ); - Ok(canonical_carrier_wire) - } - } - } - - fn lock_candidate( - locks: &mut BTreeMap, - target: BlockReference, - kind: LockKindV1, - ) -> Result<(), JournalErrorV1> { - let slot = RbcSlotKeyV1::of(target); - Self::lock_exact( - locks, - slot, - target, - JournalErrorV1::ConflictingPhaseLock { kind, slot }, - ) - } - - fn lock_consensus_slot( - &mut self, - consensus_round: RoundNumber, - enclosing_carrier: BlockReference, - ) -> Result<(), JournalErrorV1> { - if consensus_round == 0 { - return Err(JournalErrorV1::EncodedGenesisConsensusVertex); - } - if enclosing_carrier.authority != self.own_authority - || !self - .own_carriers - .values() - .any(|reference| *reference == enclosing_carrier) - { - return Err(JournalErrorV1::ConsensusCarrierNotFixed(enclosing_carrier)); - } - let matching_vertex = self - .outbound - .get(&enclosing_carrier) - .and_then(|outbound| outbound.candidate.header().consensus_vertex()) - .is_some_and(|vertex| vertex.consensus_round() == consensus_round); - if !matching_vertex { - return Err(JournalErrorV1::ConsensusVertexMismatch { - consensus_round, - enclosing_carrier, - }); - } - Self::lock_exact( - &mut self.consensus_slots, - consensus_round, - enclosing_carrier, - JournalErrorV1::ConflictingConsensusSlot(consensus_round), - ) - } - - fn lock_leader_choice( - &mut self, - consensus_round: RoundNumber, - choice: LeaderChoiceV1, - ) -> Result<(), JournalErrorV1> { - let Some(enclosing_carrier) = self.consensus_slot(consensus_round) else { - return Err(JournalErrorV1::LeaderChoiceWithoutConsensusSlot( - consensus_round, - )); - }; - let matching_choice = self - .outbound - .get(&enclosing_carrier) - .and_then(|outbound| outbound.candidate.header().consensus_vertex()) - .is_some_and(|vertex| { - vertex.consensus_round() == consensus_round && vertex.leader_choice() == choice - }); - if !matching_choice { - return Err(JournalErrorV1::LeaderChoiceCandidateMismatch( - consensus_round, - )); - } - Self::lock_exact( - &mut self.leader_choices, - consensus_round, - choice, - JournalErrorV1::ConflictingLeaderChoice(consensus_round), - ) - } - - fn persist_outbound_content( - &mut self, - candidate: &CandidateCarrierV1, - ) -> Result<(), JournalErrorV1> { - let reference = candidate.reference(); - if reference.authority != self.own_authority { - return Err(JournalErrorV1::OwnCarrierAuthorMismatch { - expected: self.own_authority, - actual: reference.authority, - }); - } - if reference.round == 0 { - return Err(JournalErrorV1::EncodedGenesisCarrier); - } - if self - .own_carrier(reference.round) - .is_some_and(|fixed| fixed != reference) - { - return Err(JournalErrorV1::ConflictingOwnCarrier(reference.round)); - } - let canonical_carrier_wire = self.retain_carrier_content(candidate)?; - match self.outbound.get(&reference) { - Some(existing) - if existing.candidate != *candidate - || existing.canonical_carrier_wire != canonical_carrier_wire => - { - Err(JournalErrorV1::ConflictingOutboundContent(reference)) - } - Some(_) => Ok(()), - None => { - self.outbound.insert( - reference, - PartialOutboundCarrierV1::new(candidate.clone(), canonical_carrier_wire), - ); - Ok(()) - } - } - } - - fn persist_outbound_sidecar( - &mut self, - authenticated: &LocallyAuthenticatedCarrierV1, - ) -> Result<(), JournalErrorV1> { - if authenticated.context() != self.context { - return Err(JournalErrorV1::OutboundAuthenticationContextMismatch); - } - let reference = authenticated.reference(); - if reference.authority != self.own_authority { - return Err(JournalErrorV1::OwnCarrierAuthorMismatch { - expected: self.own_authority, - actual: reference.authority, - }); - } - if self.own_carrier(reference.round) != Some(reference) { - return Err(JournalErrorV1::OwnCarrierNotFixed(reference)); - } - let Some(outbound) = self.outbound.get_mut(&reference) else { - return Err(JournalErrorV1::OutboundContentNotPersisted(reference)); - }; - if outbound.candidate != *authenticated.candidate() { - return Err(JournalErrorV1::OutboundAuthenticationCandidateMismatch( - reference, - )); - } - let authentication_sidecar = authenticated.authentication().canonical_wire_bytes(); - match &outbound.authentication_sidecar { - Some(existing) if *existing != authentication_sidecar => { - Err(JournalErrorV1::ConflictingOutboundSidecar(reference)) - } - Some(_) => Ok(()), - None => { - outbound.authentication_sidecar = Some(authentication_sidecar); - Ok(()) - } - } - } - - fn expose_outbound(&mut self, reference: BlockReference) -> Result<(), JournalErrorV1> { - if self.own_carrier(reference.round) != Some(reference) { - return Err(JournalErrorV1::OwnCarrierNotFixed(reference)); - } - let Some(outbound) = self.outbound.get(&reference) else { - return Err(JournalErrorV1::OutboundContentNotPersisted(reference)); - }; - if outbound.authentication_sidecar.is_none() { - return Err(JournalErrorV1::OutboundSidecarNotPersisted(reference)); - } - let candidate = outbound.candidate.clone(); - if self.echo_lock(RbcSlotKeyV1::of(reference)) != Some(reference) { - return Err(JournalErrorV1::OutboundEchoNotLocked(reference)); - } - for statement in candidate.header().phase_batch() { - let target = statement.target(); - let lock = match statement { - RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), - RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), - }; - if lock != Some(target) { - return Err(JournalErrorV1::OutboundPhaseNotLocked(*statement)); - } - } - if let Some(vertex) = candidate.header().consensus_vertex() { - let consensus_round = vertex.consensus_round(); - if self.consensus_slot(consensus_round) != Some(reference) { - return Err(JournalErrorV1::OutboundConsensusSlotNotLocked { - consensus_round, - reference, - }); - } - if self.leader_choice(consensus_round) != Some(vertex.leader_choice()) { - return Err(JournalErrorV1::OutboundLeaderChoiceNotLocked( - consensus_round, - )); - } - } - self.outbound - .get_mut(&reference) - .expect("outbound remains retained") - .exposed = true; - Ok(()) - } - - fn apply_phase_statement( - &mut self, - outer: BlockReference, - index: usize, - sender: AuthorityIndex, - statement: RbcPhaseStatementV1, - ) -> Result<(), JournalErrorV1> { - if sender != outer.authority { - return Err(JournalErrorV1::PhaseSenderMismatch { - outer_author: outer.authority, - sender, - }); - } - let target = statement.target(); - if target.round >= outer.round { - return Err(JournalErrorV1::PhaseTargetNotOlder { outer, target }); - } - let Some(retained_outer) = self.retained_carriers.get(&outer) else { - return Err(JournalErrorV1::OuterCarrierContentNotRetained(outer)); - }; - if retained_outer - .candidate - .header() - .phase_batch() - .get(index) - .copied() - != Some(statement) - { - return Err(JournalErrorV1::PhaseBatchEntryMismatch { outer, index }); - } - let outer_slot = RbcSlotKeyV1::of(outer); - let authorized = self.own_carrier(outer.round) == Some(outer) - || self.admission_lock(outer_slot) == Some(outer) - || self.delivery_lock(outer_slot) == Some(outer); - if !authorized { - return Err(JournalErrorV1::OuterCarrierNotAdmittedOrDelivered(outer)); - } - - let cursor = self.phase_batch_cursor(outer); - if index > cursor { - return Err(JournalErrorV1::PhaseBatchIndexGap { - outer, - expected: cursor, - actual: index, - }); - } - if let Some(existing) = self.applied_phase_entries.get(&(outer, index)) { - return if existing.sender == sender && existing.statement == statement { - Ok(()) - } else { - Err(JournalErrorV1::ConflictingPhaseBatchEntry { outer, index }) - }; - } - if index < cursor { - return Err(JournalErrorV1::MissingAppliedPhaseEntry { outer, index }); - } - - if sender == self.own_authority { - if self.own_carrier(outer.round) != Some(outer) { - return Err(JournalErrorV1::OwnOuterCarrierNotFixed(outer)); - } - let lock = match statement { - RbcPhaseStatementV1::Echo { .. } => self.echo_lock(RbcSlotKeyV1::of(target)), - RbcPhaseStatementV1::Ready { .. } => self.ready_lock(RbcSlotKeyV1::of(target)), - }; - if lock != Some(target) { - return Err(JournalErrorV1::OwnPhaseWithoutDurableLock(statement)); - } - } - - let sender_key = SenderPhaseKeyV1 { - slot: RbcSlotKeyV1::of(target), - sender, - phase: PhaseKindV1::of(statement), - }; - let outcome = match self.sender_phase_locks.get(&sender_key) { - Some(existing) if *existing != target => AppliedPhaseOutcomeV1::IgnoredEquivocation, - Some(_) => AppliedPhaseOutcomeV1::IgnoredReplay, - None => { - self.sender_phase_locks.insert(sender_key, target); - AppliedPhaseOutcomeV1::Counted - } - }; - self.applied_phase_entries.insert( - (outer, index), - AppliedPhaseEntryV1 { - sender, - statement, - outcome, - }, - ); - Ok(()) - } - - fn advance_phase_cursor( - &mut self, - outer: BlockReference, - index: usize, - ) -> Result<(), JournalErrorV1> { - let cursor = self.phase_batch_cursor(outer); - if index < cursor { - return if self.applied_phase_entries.contains_key(&(outer, index)) { - Ok(()) - } else { - Err(JournalErrorV1::MissingAppliedPhaseEntry { outer, index }) - }; - } - if index > cursor { - return Err(JournalErrorV1::PhaseBatchIndexGap { - outer, - expected: cursor, - actual: index, - }); - } - if !self.applied_phase_entries.contains_key(&(outer, index)) { - return Err(JournalErrorV1::PhaseCursorBeforeApplication { outer, index }); - } - self.phase_batch_cursors.insert(outer, cursor + 1); - Ok(()) - } - - fn lock_exact( - map: &mut BTreeMap, - key: K, - value: V, - conflict: JournalErrorV1, - ) -> Result<(), JournalErrorV1> - where - K: Ord, - V: Eq, - { - match map.get(&key) { - Some(existing) if *existing != value => Err(conflict), - Some(_) => Ok(()), - None => { - map.insert(key, value); - Ok(()) - } - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum LockKindV1 { - Admission, - Echo, - Ready, - Delivery, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum JournalErrorV1 { - ContextMismatch, - IngressSequence { - expected: u64, - actual: u64, - }, - AuthenticatedIngressContextMismatch, - AuthenticatedIngressReceiverMismatch { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - CandidateCommitteeMismatch, - CanonicalCarrierEncoding(BlockReference), - CarrierContentNotRetained(BlockReference), - ConflictingRetainedContent(BlockReference), - AdmissionWithoutAuthenticatedIngress(BlockReference), - EchoWithoutAdmission(BlockReference), - OwnCarrierAuthorMismatch { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - EncodedGenesisCarrier, - ConflictingOwnCarrier(RoundNumber), - ConflictingPhaseLock { - kind: LockKindV1, - slot: RbcSlotKeyV1, - }, - DeliveryWithoutMatchingReady(BlockReference), - EncodedGenesisConsensusVertex, - ConsensusCarrierNotFixed(BlockReference), - ConsensusVertexMismatch { - consensus_round: RoundNumber, - enclosing_carrier: BlockReference, - }, - ConflictingConsensusSlot(RoundNumber), - LeaderChoiceWithoutConsensusSlot(RoundNumber), - LeaderChoiceCandidateMismatch(RoundNumber), - ConflictingLeaderChoice(RoundNumber), - OwnCarrierNotFixed(BlockReference), - ConflictingOutboundContent(BlockReference), - OutboundContentNotPersisted(BlockReference), - ConflictingOutboundSidecar(BlockReference), - OutboundSidecarNotPersisted(BlockReference), - OutboundAuthenticationContextMismatch, - OutboundAuthenticationCandidateMismatch(BlockReference), - OutboundEchoNotLocked(BlockReference), - OutboundPhaseNotLocked(RbcPhaseStatementV1), - OutboundConsensusSlotNotLocked { - consensus_round: RoundNumber, - reference: BlockReference, - }, - OutboundLeaderChoiceNotLocked(RoundNumber), - PhaseSenderMismatch { - outer_author: AuthorityIndex, - sender: AuthorityIndex, - }, - PhaseTargetNotOlder { - outer: BlockReference, - target: BlockReference, - }, - OuterCarrierContentNotRetained(BlockReference), - OuterCarrierNotAdmittedOrDelivered(BlockReference), - OwnOuterCarrierNotFixed(BlockReference), - PhaseBatchEntryMismatch { - outer: BlockReference, - index: usize, - }, - PhaseBatchIndexGap { - outer: BlockReference, - expected: usize, - actual: usize, - }, - ConflictingPhaseBatchEntry { - outer: BlockReference, - index: usize, - }, - MissingAppliedPhaseEntry { - outer: BlockReference, - index: usize, - }, - OwnPhaseWithoutDurableLock(RbcPhaseStatementV1), - PhaseCursorBeforeApplication { - outer: BlockReference, - index: usize, - }, -} - -impl fmt::Display for JournalErrorV1 { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "Starfish-RBC-DAG journal error: {self:?}") - } -} - -impl Error for JournalErrorV1 {} - -/// A deterministic write-ahead journal with a volatile replayed snapshot. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct WriteAheadJournalV1 { - context: RbcDagContextV1, - own_authority: AuthorityIndex, - durable_events: Vec, - snapshot: JournalSnapshotV1, -} - -impl WriteAheadJournalV1 { - pub fn new(context: RbcDagContextV1, own_authority: AuthorityIndex) -> Self { - Self { - context, - own_authority, - durable_events: Vec::new(), - snapshot: JournalSnapshotV1::new(context, own_authority), - } - } - - /// Validate against a cloned snapshot, then atomically make the event - /// durable and visible. A real backend maps this boundary to its durable - /// transaction commit. - pub fn append(&mut self, event: JournalEventV1) -> Result<(), JournalErrorV1> { - let mut next = self.snapshot.clone(); - next.apply(&event)?; - self.durable_events.push(event); - self.snapshot = next; - Ok(()) - } - - pub fn record_authenticated_ingress( - &mut self, - authenticated: AuthenticatedCarrierV1, - provenance: IngressProvenanceV1, - ) -> Result { - let sequence = self.snapshot.next_ingress_sequence(); - self.append(JournalEventV1::AuthenticatedIngress { - context: self.context, - sequence, - authenticated, - provenance, - })?; - Ok(sequence) - } - - pub fn durable_events(&self) -> &[JournalEventV1] { - &self.durable_events - } - - pub fn snapshot(&self) -> &JournalSnapshotV1 { - &self.snapshot - } - - /// Rebuild volatile state in exact durable order. - pub fn restart(&self) -> Result { - Self::from_durable_events( - self.context, - self.own_authority, - self.durable_events.clone(), - ) - } - - pub fn from_durable_events( - context: RbcDagContextV1, - own_authority: AuthorityIndex, - durable_events: Vec, - ) -> Result { - let mut snapshot = JournalSnapshotV1::new(context, own_authority); - for event in &durable_events { - snapshot.apply(event)?; - } - Ok(Self { - context, - own_authority, - durable_events, - snapshot, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - committee::Committee, - crypto::{TransactionsCommitment, mac_keyrings_for_test}, - starfish_rbc_dag::{ - CarrierAuthenticationV1, CarrierAuthorizerV1, CarrierHeaderV1Args, - ConsensusVertexReference, ConsensusVertexV1, RbcDagProtocolInstanceId, - carrier_genesis_reference, - }, - types::{BlockAuthenticationScheme, BlockDigest}, - }; - - fn committee() -> std::sync::Arc { - Committee::new_test(vec![1; 4]) - } - - fn context(marker: u8) -> RbcDagContextV1 { - let committee = committee(); - RbcDagContextV1::new( - RbcDagProtocolInstanceId::new([marker; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ) - .unwrap() - } - - fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { - BlockReference { - authority, - round, - digest: BlockDigest::from([marker; 32]), - } - } - - fn journal() -> WriteAheadJournalV1 { - WriteAheadJournalV1::new(context(0xA1), 1) - } - - fn fix_event(journal: &WriteAheadJournalV1, reference: BlockReference) -> JournalEventV1 { - JournalEventV1::FixOwnCarrier { - context: journal.context, - reference, - } - } - - fn candidate( - author: AuthorityIndex, - round: RoundNumber, - marker: u8, - phase_batch: Vec, - consensus_vertex: Option, - ) -> CandidateCarrierV1 { - let committee = committee(); - let previous_round = round - 1; - let parent = |authority| { - if previous_round == 0 { - carrier_genesis_reference(authority) - } else { - reference(authority, previous_round, 0x40 + authority as u8) - } - }; - let mut parent_stake = committee.get_stake(author).unwrap(); - let mut weak_parents = Vec::new(); - for authority in committee - .authorities() - .filter(|authority| *authority != author) - { - if parent_stake >= committee.quorum_threshold() { - break; - } - parent_stake += committee.get_stake(authority).unwrap(); - weak_parents.push(parent(authority)); - } - CandidateCarrierV1::try_new( - CarrierHeaderV1Args { - author, - carrier_round: round, - own_prev: parent(author), - weak_parents, - transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - data_acknowledgments: Vec::new(), - phase_batch, - consensus_vertex, - creation_time_ns: u64::from(marker), - }, - &committee, - ) - .unwrap() - } - - fn consensus_vertex( - enclosing_author: AuthorityIndex, - consensus_round: RoundNumber, - ) -> ConsensusVertexV1 { - let committee = committee(); - let parent_round = consensus_round - 1; - let parent = |authority| { - let carrier = if parent_round == 0 { - carrier_genesis_reference(authority) - } else { - reference(authority, parent_round, 0xD0 + authority as u8) - }; - ConsensusVertexReference::new(carrier, parent_round) - }; - let strong_parents: Vec<_> = committee.authorities().map(parent).collect(); - assert!( - strong_parents - .iter() - .any(|parent| parent.author() == enclosing_author) - ); - let leader = parent(committee.elect_leader(parent_round)); - ConsensusVertexV1::new( - consensus_round, - strong_parents, - vec![None; committee.len()], - LeaderChoiceV1::Vote { leader }, - ) - } - - fn outbound_content_event( - journal: &WriteAheadJournalV1, - candidate: &CandidateCarrierV1, - ) -> JournalEventV1 { - JournalEventV1::PersistOutboundContent { - context: journal.context, - candidate: candidate.clone(), - } - } - - fn persist_and_fix(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { - journal - .append(outbound_content_event(journal, candidate)) - .unwrap(); - journal - .append(fix_event(journal, candidate.reference())) - .unwrap(); - } - - fn retain_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { - journal - .append(JournalEventV1::RetainCandidateContent { - context: journal.context, - candidate: candidate.clone(), - }) - .unwrap(); - } - - fn authentication_with_marker( - candidate: &CandidateCarrierV1, - context_marker: u8, - ) -> CarrierAuthenticationV1 { - let committee = committee(); - let context = context(context_marker); - let keyrings = mac_keyrings_for_test(committee.len()); - let author = candidate.header().author() as usize; - context - .authenticate( - candidate, - &committee, - CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ) - .unwrap() - } - - fn locally_authenticated_with_marker( - candidate: &CandidateCarrierV1, - context_marker: u8, - ) -> LocallyAuthenticatedCarrierV1 { - let committee = committee(); - let context = context(context_marker); - let keyrings = mac_keyrings_for_test(committee.len()); - let author = candidate.header().author() as usize; - context - .authenticate_local( - candidate.clone(), - &committee, - CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ) - .unwrap() - } - - fn locally_authenticated(candidate: &CandidateCarrierV1) -> LocallyAuthenticatedCarrierV1 { - locally_authenticated_with_marker(candidate, 0xA1) - } - - fn authenticated_with_marker( - candidate: &CandidateCarrierV1, - receiver: AuthorityIndex, - context_marker: u8, - ) -> AuthenticatedCarrierV1 { - let committee = committee(); - let context = context(context_marker); - let keyrings = mac_keyrings_for_test(committee.len()); - context - .verify_authentication( - candidate.clone(), - authentication_with_marker(candidate, context_marker), - receiver, - &committee, - &keyrings[receiver as usize], - ) - .unwrap() - } - - fn authenticated( - candidate: &CandidateCarrierV1, - receiver: AuthorityIndex, - ) -> AuthenticatedCarrierV1 { - authenticated_with_marker(candidate, receiver, 0xA1) - } - - fn authenticate_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { - journal - .record_authenticated_ingress( - authenticated(candidate, journal.own_authority), - IngressProvenanceV1::DirectFromAuthor, - ) - .unwrap(); - } - - fn admit_candidate(journal: &mut WriteAheadJournalV1, candidate: &CandidateCarrierV1) { - authenticate_candidate(journal, candidate); - journal - .append(JournalEventV1::LockAdmission { - context: journal.context, - target: candidate.reference(), - }) - .unwrap(); - } - - fn prepare_outbound_for_exposure( - journal: &mut WriteAheadJournalV1, - candidate: &CandidateCarrierV1, - ) { - let reference = candidate.reference(); - persist_and_fix(journal, candidate); - journal - .append(JournalEventV1::LockEcho { - context: journal.context, - target: reference, - }) - .unwrap(); - journal - .append(JournalEventV1::PersistOutboundSidecar { - context: journal.context, - authenticated: locally_authenticated(candidate), - }) - .unwrap(); - } - - fn assert_before_after(journal: &WriteAheadJournalV1, event: JournalEventV1, assertion: F) - where - F: Fn(&JournalSnapshotV1) -> bool, - { - let before = journal.restart().unwrap(); - assert!(!assertion(before.snapshot())); - - let mut after = journal.clone(); - after.append(event).unwrap(); - let after = after.restart().unwrap(); - assert!(assertion(after.snapshot())); - } - - #[test] - fn authenticated_ingress_sequence_and_bytes_survive_restart_in_order() { - let mut journal = journal(); - let first_candidate = candidate(0, 1, 0x10, Vec::new(), None); - let second_candidate = candidate(2, 1, 0x20, Vec::new(), None); - let first = first_candidate.reference(); - let second = second_candidate.reference(); - let first_authenticated = authenticated(&first_candidate, 1); - let second_authenticated = authenticated(&second_candidate, 1); - let first_wire = first_candidate.canonical_wire_bytes().unwrap(); - let second_sidecar = second_authenticated.authentication().canonical_wire_bytes(); - assert_eq!( - journal - .record_authenticated_ingress( - first_authenticated, - IngressProvenanceV1::DirectFromAuthor, - ) - .unwrap(), - 0 - ); - assert_eq!( - journal - .record_authenticated_ingress( - second_authenticated, - IngressProvenanceV1::Relayed { peer: 3 }, - ) - .unwrap(), - 1 - ); - - let restarted = journal.restart().unwrap().restart().unwrap(); - let ingress = restarted.snapshot().authenticated_ingress(); - assert_eq!( - ingress - .iter() - .map(|entry| entry.sequence()) - .collect::>(), - [0, 1] - ); - assert_eq!(ingress[0].reference(), first); - assert_eq!(ingress[1].reference(), second); - assert_eq!(ingress[0].canonical_carrier_wire(), first_wire); - assert_eq!(ingress[1].authentication_sidecar(), second_sidecar); - - let mut corrupt = journal.durable_events().to_vec(); - if let JournalEventV1::AuthenticatedIngress { sequence, .. } = &mut corrupt[1] { - *sequence = 2; - } - assert!(matches!( - WriteAheadJournalV1::from_durable_events(context(0xA1), 1, corrupt), - Err(JournalErrorV1::IngressSequence { - expected: 1, - actual: 2 - }) - )); - } - - #[test] - fn crash_boundaries_preserve_each_slot_global_lock() { - let own_candidate = candidate(1, 1, 0x11, Vec::new(), None); - let target_candidate = candidate(2, 1, 0x21, Vec::new(), None); - let own = own_candidate.reference(); - let target = target_candidate.reference(); - let slot = RbcSlotKeyV1::of(target); - let mut content_base = journal(); - content_base - .append(outbound_content_event(&content_base, &own_candidate)) - .unwrap(); - assert_before_after(&content_base, fix_event(&content_base, own), |state| { - state.own_carrier(1) == Some(own) - }); - - let mut base = content_base; - base.append(fix_event(&base, own)).unwrap(); - authenticate_candidate(&mut base, &target_candidate); - let admission = JournalEventV1::LockAdmission { - context: base.context, - target, - }; - assert_before_after(&base, admission.clone(), |state| { - state.admission_lock(slot) == Some(target) - }); - base.append(admission).unwrap(); - - let echo = JournalEventV1::LockEcho { - context: base.context, - target, - }; - assert_before_after(&base, echo, |state| state.echo_lock(slot) == Some(target)); - - let mut ready_base = base.clone(); - let ready = JournalEventV1::LockReady { - context: base.context, - target, - }; - assert_before_after(&ready_base, ready.clone(), |state| { - state.ready_lock(slot) == Some(target) - }); - ready_base.append(ready).unwrap(); - - let delivery = JournalEventV1::LockDelivery { - context: base.context, - target, - }; - assert_before_after(&ready_base, delivery, |state| { - state.delivery_lock(slot) == Some(target) - }); - } - - #[test] - fn crash_boundaries_preserve_consensus_and_leader_choice_locks() { - let mut journal = journal(); - let vertex = consensus_vertex(1, 1); - let expected_choice = vertex.leader_choice(); - let own_candidate = candidate(1, 2, 0x32, Vec::new(), Some(vertex)); - let own = own_candidate.reference(); - persist_and_fix(&mut journal, &own_candidate); - let consensus = JournalEventV1::LockConsensusSlot { - context: journal.context, - consensus_round: 1, - enclosing_carrier: own, - }; - assert_before_after(&journal, consensus.clone(), |state| { - state.consensus_slot(1) == Some(own) - }); - - journal.append(consensus).unwrap(); - let choice = JournalEventV1::LockLeaderChoice { - context: journal.context, - consensus_round: 1, - choice: expected_choice, - }; - assert_before_after(&journal, choice, |state| { - state.leader_choice(1) == Some(expected_choice) - }); - } - - #[test] - fn outbound_content_sidecar_and_exposure_are_separate_crash_boundaries() { - let mut journal = journal(); - let own_candidate = candidate(1, 1, 0x41, Vec::new(), None); - let own = own_candidate.reference(); - let expected_wire = own_candidate.canonical_wire_bytes().unwrap(); - let own_authenticated = locally_authenticated(&own_candidate); - let expected_sidecar = own_authenticated.authentication().canonical_wire_bytes(); - - let content = JournalEventV1::PersistOutboundContent { - context: journal.context, - candidate: own_candidate, - }; - assert_before_after(&journal, content.clone(), |state| { - state.outbound.contains_key(&own) - }); - assert!(journal.snapshot().retransmissions().is_empty()); - journal.append(content).unwrap(); - - let fix = fix_event(&journal, own); - assert_before_after(&journal, fix.clone(), |state| { - state.own_carrier(1) == Some(own) - }); - journal.append(fix).unwrap(); - journal - .append(JournalEventV1::LockEcho { - context: journal.context, - target: own, - }) - .unwrap(); - - let sidecar = JournalEventV1::PersistOutboundSidecar { - context: journal.context, - authenticated: own_authenticated, - }; - assert_before_after(&journal, sidecar.clone(), |state| { - state.outbound(own).is_some() - }); - journal.append(sidecar).unwrap(); - assert!(journal.snapshot().retransmissions().is_empty()); - - let expose = JournalEventV1::ExposeOutbound { - context: journal.context, - reference: own, - }; - assert_before_after(&journal, expose.clone(), |state| { - state - .outbound(own) - .is_some_and(|outbound| outbound.exposed()) - }); - journal.append(expose).unwrap(); - - let before = journal.snapshot().retransmissions(); - let after = journal.restart().unwrap().snapshot().retransmissions(); - assert_eq!(after, before); - assert_eq!(after[0].canonical_carrier_wire(), expected_wire); - assert_eq!(after[0].authentication_sidecar(), expected_sidecar); - } - - #[test] - fn outbound_cannot_be_exposed_before_both_exact_records_are_durable() { - let mut journal = journal(); - let own_candidate = candidate(1, 1, 0x51, Vec::new(), None); - let own = own_candidate.reference(); - let context = journal.context; - let expose = || JournalEventV1::ExposeOutbound { - context, - reference: own, - }; - assert_eq!( - journal.append(expose()).unwrap_err(), - JournalErrorV1::OwnCarrierNotFixed(own) - ); - journal - .append(JournalEventV1::PersistOutboundContent { - context: journal.context, - candidate: own_candidate, - }) - .unwrap(); - journal.append(fix_event(&journal, own)).unwrap(); - journal - .append(JournalEventV1::LockEcho { - context: journal.context, - target: own, - }) - .unwrap(); - assert_eq!( - journal.append(expose()).unwrap_err(), - JournalErrorV1::OutboundSidecarNotPersisted(own) - ); - } - - #[test] - fn phase_application_is_durable_before_the_cursor_advances() { - let mut journal = journal(); - let statement = RbcPhaseStatementV1::Echo { - target: reference(0, 1, 0x60), - }; - let outer_candidate = candidate(2, 3, 0x62, vec![statement], None); - let outer = outer_candidate.reference(); - admit_candidate(&mut journal, &outer_candidate); - let apply = JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer, - index: 0, - sender: 2, - statement, - }; - assert_before_after(&journal, apply.clone(), |state| { - state.phase_statement_applied(outer, 0) && state.phase_batch_cursor(outer) == 0 - }); - journal.append(apply.clone()).unwrap(); - - let advance = JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer, - index: 0, - }; - assert_before_after(&journal, advance, |state| { - state.phase_batch_cursor(outer) == 1 - }); - - let restarted = journal.restart().unwrap(); - let mut retried = restarted.clone(); - retried.append(apply).unwrap(); - retried - .append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer, - index: 0, - }) - .unwrap(); - assert_eq!(retried.snapshot().phase_batch_cursor(outer), 1); - } - - #[test] - fn only_admitted_conflict_processes_until_the_other_is_delivered() { - let mut journal = journal(); - let first_statement = RbcPhaseStatementV1::Echo { - target: reference(0, 1, 0x63), - }; - let second_statement = RbcPhaseStatementV1::Echo { - target: reference(3, 1, 0x64), - }; - let first_candidate = candidate(2, 2, 0x65, vec![first_statement], None); - let second_candidate = candidate(2, 2, 0x66, vec![second_statement], None); - let first = first_candidate.reference(); - let second = second_candidate.reference(); - authenticate_candidate(&mut journal, &first_candidate); - authenticate_candidate(&mut journal, &second_candidate); - journal - .append(JournalEventV1::LockAdmission { - context: journal.context, - target: first, - }) - .unwrap(); - assert!(matches!( - journal.append(JournalEventV1::LockAdmission { - context: journal.context, - target: second, - }), - Err(JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Admission, - .. - }) - )); - journal - .append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: first, - index: 0, - sender: 2, - statement: first_statement, - }) - .unwrap(); - assert_eq!( - journal - .append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: second, - index: 0, - sender: 2, - statement: second_statement, - }) - .unwrap_err(), - JournalErrorV1::OuterCarrierNotAdmittedOrDelivered(second) - ); - - journal - .append(JournalEventV1::LockReady { - context: journal.context, - target: second, - }) - .unwrap(); - journal - .append(JournalEventV1::LockDelivery { - context: journal.context, - target: second, - }) - .unwrap(); - journal - .append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: second, - index: 0, - sender: 2, - statement: second_statement, - }) - .unwrap(); - journal - .append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer: second, - index: 0, - }) - .unwrap(); - assert_eq!(journal.snapshot().phase_batch_cursor(second), 1); - } - - #[test] - fn cursor_cannot_skip_or_advance_before_idempotent_application() { - let mut journal = journal(); - let statement = RbcPhaseStatementV1::Ready { - target: reference(0, 1, 0x70), - }; - let second_statement = RbcPhaseStatementV1::Ready { - target: reference(3, 1, 0x71), - }; - let outer_candidate = candidate(2, 3, 0x72, vec![statement, second_statement], None); - let outer = outer_candidate.reference(); - admit_candidate(&mut journal, &outer_candidate); - assert!(matches!( - journal.append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer, - index: 0, - }), - Err(JournalErrorV1::PhaseCursorBeforeApplication { .. }) - )); - assert!(matches!( - journal.append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer, - index: 1, - sender: 2, - statement: second_statement, - }), - Err(JournalErrorV1::PhaseBatchIndexGap { - expected: 0, - actual: 1, - .. - }) - )); - } - - #[test] - fn own_embedded_phase_requires_lock_to_precede_it_in_the_log() { - let mut journal = journal(); - let target_candidate = candidate(0, 1, 0x80, Vec::new(), None); - let target = target_candidate.reference(); - let statement = RbcPhaseStatementV1::Echo { target }; - let outer_candidate = candidate(1, 2, 0x81, vec![statement], None); - let outer = outer_candidate.reference(); - admit_candidate(&mut journal, &target_candidate); - persist_and_fix(&mut journal, &outer_candidate); - let apply = JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer, - index: 0, - sender: 1, - statement, - }; - assert_eq!( - journal.append(apply.clone()).unwrap_err(), - JournalErrorV1::OwnPhaseWithoutDurableLock(statement) - ); - journal - .append(JournalEventV1::LockEcho { - context: journal.context, - target, - }) - .unwrap(); - journal.append(apply).unwrap(); - - let mut reversed = journal.durable_events().to_vec(); - let last = reversed.len() - 1; - reversed.swap(last - 1, last); - assert_eq!( - WriteAheadJournalV1::from_durable_events(journal.context, 1, reversed).unwrap_err(), - JournalErrorV1::OwnPhaseWithoutDurableLock(statement) - ); - } - - #[test] - fn conflicting_local_carrier_and_phase_choices_are_rejected() { - let mut journal = journal(); - let first_candidate = candidate(1, 1, 0x91, Vec::new(), None); - let second_candidate = candidate(1, 1, 0x92, Vec::new(), None); - let first_carrier = first_candidate.reference(); - let second_carrier = second_candidate.reference(); - journal - .append(outbound_content_event(&journal, &first_candidate)) - .unwrap(); - journal - .append(outbound_content_event(&journal, &second_candidate)) - .unwrap(); - journal.append(fix_event(&journal, first_carrier)).unwrap(); - assert_eq!( - journal - .append(fix_event(&journal, second_carrier)) - .unwrap_err(), - JournalErrorV1::ConflictingOwnCarrier(1) - ); - - let first_target_candidate = candidate(0, 1, 0x93, Vec::new(), None); - let second_target_candidate = candidate(0, 1, 0x94, Vec::new(), None); - let first_target = first_target_candidate.reference(); - let second_target = second_target_candidate.reference(); - retain_candidate(&mut journal, &first_target_candidate); - retain_candidate(&mut journal, &second_target_candidate); - journal - .append(JournalEventV1::LockReady { - context: journal.context, - target: first_target, - }) - .unwrap(); - assert!(matches!( - journal.append(JournalEventV1::LockReady { - context: journal.context, - target: second_target, - }), - Err(JournalErrorV1::ConflictingPhaseLock { - kind: LockKindV1::Ready, - .. - }) - )); - } - - #[test] - fn conflicting_remote_phase_is_durably_ignored_and_does_not_stall_cursor() { - let mut journal = journal(); - let first = reference(0, 1, 0xA0); - let second = reference(0, 1, 0xA1); - let first_statement = RbcPhaseStatementV1::Ready { target: first }; - let second_statement = RbcPhaseStatementV1::Ready { target: second }; - let outer_one_candidate = candidate(2, 2, 0xA2, vec![first_statement], None); - let outer_two_candidate = candidate(2, 3, 0xA3, vec![second_statement], None); - let outer_one = outer_one_candidate.reference(); - let outer_two = outer_two_candidate.reference(); - admit_candidate(&mut journal, &outer_one_candidate); - admit_candidate(&mut journal, &outer_two_candidate); - journal - .append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: outer_one, - index: 0, - sender: 2, - statement: first_statement, - }) - .unwrap(); - journal - .append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer: outer_one, - index: 0, - }) - .unwrap(); - journal - .append(JournalEventV1::ApplyPhaseStatement { - context: journal.context, - outer: outer_two, - index: 0, - sender: 2, - statement: second_statement, - }) - .unwrap(); - assert_eq!( - journal.snapshot().phase_statement_outcome(outer_two, 0), - Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) - ); - journal - .append(JournalEventV1::AdvancePhaseBatchCursor { - context: journal.context, - outer: outer_two, - index: 0, - }) - .unwrap(); - assert_eq!(journal.snapshot().phase_batch_cursor(outer_two), 1); - assert_eq!( - journal - .restart() - .unwrap() - .snapshot() - .phase_statement_outcome(outer_two, 0), - Some(AppliedPhaseOutcomeV1::IgnoredEquivocation) - ); - } - - #[test] - fn replay_is_idempotent_and_foreign_namespace_fails_closed() { - let mut journal = journal(); - let own_candidate = candidate(1, 1, 0xB1, Vec::new(), None); - let own = own_candidate.reference(); - persist_and_fix(&mut journal, &own_candidate); - journal - .append(JournalEventV1::LockEcho { - context: journal.context, - target: own, - }) - .unwrap(); - let once = journal.restart().unwrap(); - let twice = once.restart().unwrap(); - assert_eq!(once.snapshot(), twice.snapshot()); - assert_eq!(once.durable_events(), twice.durable_events()); - - let foreign = JournalEventV1::LockReady { - context: context(0xB2), - target: own, - }; - assert!(matches!( - journal.append(foreign), - Err(JournalErrorV1::ContextMismatch) - )); - } - - #[test] - fn recovered_content_is_durable_before_ready_but_does_not_authorize_echo() { - let mut journal = journal(); - let target_candidate = candidate(0, 1, 0xB3, Vec::new(), None); - let target = target_candidate.reference(); - let ready = JournalEventV1::LockReady { - context: journal.context, - target, - }; - assert_eq!( - journal.append(ready.clone()).unwrap_err(), - JournalErrorV1::CarrierContentNotRetained(target) - ); - - let retain = JournalEventV1::RetainCandidateContent { - context: journal.context, - candidate: target_candidate.clone(), - }; - let expected_wire = target_candidate.canonical_wire_bytes().unwrap(); - assert_before_after(&journal, retain.clone(), |state| { - state.retained_carrier(target) == Some(expected_wire.as_slice()) - }); - journal.append(retain).unwrap(); - assert_before_after(&journal, ready.clone(), |state| { - state.ready_lock(RbcSlotKeyV1::of(target)) == Some(target) - }); - journal.append(ready).unwrap(); - - let echo = JournalEventV1::LockEcho { - context: journal.context, - target, - }; - assert_eq!( - journal.append(echo.clone()).unwrap_err(), - JournalErrorV1::EchoWithoutAdmission(target) - ); - admit_candidate(&mut journal, &target_candidate); - journal.append(echo).unwrap(); - } - - #[test] - fn own_fix_and_echo_cannot_precede_exact_typed_content() { - let mut journal = journal(); - let own_candidate = candidate(1, 1, 0xB4, Vec::new(), None); - let own = own_candidate.reference(); - assert_eq!( - journal.append(fix_event(&journal, own)).unwrap_err(), - JournalErrorV1::OutboundContentNotPersisted(own) - ); - - let content = outbound_content_event(&journal, &own_candidate); - let expected_wire = own_candidate.canonical_wire_bytes().unwrap(); - assert_before_after(&journal, content.clone(), |state| { - state.retained_carrier(own) == Some(expected_wire.as_slice()) - && state.own_carrier(1).is_none() - }); - journal.append(content).unwrap(); - let echo = JournalEventV1::LockEcho { - context: journal.context, - target: own, - }; - assert_eq!( - journal.append(echo.clone()).unwrap_err(), - JournalErrorV1::EchoWithoutAdmission(own) - ); - journal.append(fix_event(&journal, own)).unwrap(); - journal.append(echo).unwrap(); - } - - #[test] - fn outbound_exposure_waits_for_every_embedded_phase_lock() { - let mut journal = journal(); - let echo_target_candidate = candidate(0, 1, 0xB5, Vec::new(), None); - let ready_target_candidate = candidate(2, 1, 0xB6, Vec::new(), None); - let echo = RbcPhaseStatementV1::Echo { - target: echo_target_candidate.reference(), - }; - let ready = RbcPhaseStatementV1::Ready { - target: ready_target_candidate.reference(), - }; - let own_candidate = candidate(1, 2, 0xB7, vec![echo, ready], None); - let own = own_candidate.reference(); - admit_candidate(&mut journal, &echo_target_candidate); - retain_candidate(&mut journal, &ready_target_candidate); - prepare_outbound_for_exposure(&mut journal, &own_candidate); - let expose = JournalEventV1::ExposeOutbound { - context: journal.context, - reference: own, - }; - - assert_eq!( - journal.append(expose.clone()).unwrap_err(), - JournalErrorV1::OutboundPhaseNotLocked(echo) - ); - journal - .append(JournalEventV1::LockEcho { - context: journal.context, - target: echo.target(), - }) - .unwrap(); - assert_eq!( - journal.append(expose.clone()).unwrap_err(), - JournalErrorV1::OutboundPhaseNotLocked(ready) - ); - journal - .append(JournalEventV1::LockReady { - context: journal.context, - target: ready.target(), - }) - .unwrap(); - journal.append(expose).unwrap(); - } - - #[test] - fn outbound_exposure_waits_for_matching_consensus_and_leader_locks() { - let mut journal = journal(); - let vertex = consensus_vertex(1, 1); - let choice = vertex.leader_choice(); - let own_candidate = candidate(1, 2, 0xB8, Vec::new(), Some(vertex)); - let own = own_candidate.reference(); - prepare_outbound_for_exposure(&mut journal, &own_candidate); - let expose = JournalEventV1::ExposeOutbound { - context: journal.context, - reference: own, - }; - assert_eq!( - journal.append(expose.clone()).unwrap_err(), - JournalErrorV1::OutboundConsensusSlotNotLocked { - consensus_round: 1, - reference: own, - } - ); - journal - .append(JournalEventV1::LockConsensusSlot { - context: journal.context, - consensus_round: 1, - enclosing_carrier: own, - }) - .unwrap(); - assert_eq!( - journal.append(expose.clone()).unwrap_err(), - JournalErrorV1::OutboundLeaderChoiceNotLocked(1) - ); - journal - .append(JournalEventV1::LockLeaderChoice { - context: journal.context, - consensus_round: 1, - choice, - }) - .unwrap(); - journal.append(expose).unwrap(); - } - - #[test] - fn mismatched_consensus_or_leader_lock_cannot_poison_a_slot() { - let mut no_vertex_journal = journal(); - let no_vertex = candidate(1, 2, 0xB9, Vec::new(), None); - prepare_outbound_for_exposure(&mut no_vertex_journal, &no_vertex); - assert!(matches!( - no_vertex_journal.append(JournalEventV1::LockConsensusSlot { - context: no_vertex_journal.context, - consensus_round: 1, - enclosing_carrier: no_vertex.reference(), - }), - Err(JournalErrorV1::ConsensusVertexMismatch { .. }) - )); - - let mut journal = journal(); - let vertex = consensus_vertex(1, 1); - let choice = vertex.leader_choice(); - let own_candidate = candidate(1, 2, 0xBA, Vec::new(), Some(vertex)); - prepare_outbound_for_exposure(&mut journal, &own_candidate); - assert_eq!( - journal - .append(JournalEventV1::LockLeaderChoice { - context: journal.context, - consensus_round: 1, - choice, - }) - .unwrap_err(), - JournalErrorV1::LeaderChoiceWithoutConsensusSlot(1) - ); - assert!(matches!( - journal.append(JournalEventV1::LockConsensusSlot { - context: journal.context, - consensus_round: 2, - enclosing_carrier: own_candidate.reference(), - }), - Err(JournalErrorV1::ConsensusVertexMismatch { .. }) - )); - journal - .append(JournalEventV1::LockConsensusSlot { - context: journal.context, - consensus_round: 1, - enclosing_carrier: own_candidate.reference(), - }) - .unwrap(); - let wrong_choice = LeaderChoiceV1::NoVote { - leader_author: 3, - leader_round: 0, - }; - assert_eq!( - journal - .append(JournalEventV1::LockLeaderChoice { - context: journal.context, - consensus_round: 1, - choice: wrong_choice, - }) - .unwrap_err(), - JournalErrorV1::LeaderChoiceCandidateMismatch(1) - ); - journal - .append(JournalEventV1::LockLeaderChoice { - context: journal.context, - consensus_round: 1, - choice, - }) - .unwrap(); - } - - #[test] - fn authenticated_ingress_capability_is_context_and_receiver_bound() { - let candidate = candidate(0, 1, 0xBB, Vec::new(), None); - let mut journal = journal(); - assert!(matches!( - journal.append(JournalEventV1::AuthenticatedIngress { - context: journal.context, - sequence: 0, - authenticated: authenticated(&candidate, 2), - provenance: IngressProvenanceV1::Relayed { peer: 3 }, - }), - Err(JournalErrorV1::AuthenticatedIngressReceiverMismatch { - expected: 1, - actual: 2, - }) - )); - assert_eq!(journal.snapshot().next_ingress_sequence(), 0); - assert_eq!( - journal - .append(JournalEventV1::AuthenticatedIngress { - context: journal.context, - sequence: 0, - authenticated: authenticated_with_marker(&candidate, 1, 0xBC), - provenance: IngressProvenanceV1::DirectFromAuthor, - }) - .unwrap_err(), - JournalErrorV1::AuthenticatedIngressContextMismatch - ); - assert_eq!(journal.snapshot().next_ingress_sequence(), 0); - } - - #[test] - fn typed_outbound_content_rederives_bytes_and_rejects_foreign_context_sidecar() { - let mut journal = journal(); - let own_candidate = candidate(1, 1, 0xC1, Vec::new(), None); - let own = own_candidate.reference(); - let expected_wire = own_candidate.canonical_wire_bytes().unwrap(); - journal - .append(JournalEventV1::PersistOutboundContent { - context: journal.context, - candidate: own_candidate.clone(), - }) - .unwrap(); - assert_eq!( - journal.snapshot().retained_carrier(own), - Some(expected_wire.as_slice()) - ); - journal.append(fix_event(&journal, own)).unwrap(); - journal - .append(JournalEventV1::PersistOutboundSidecar { - context: journal.context, - authenticated: locally_authenticated(&own_candidate), - }) - .unwrap(); - assert_eq!( - journal - .append(JournalEventV1::PersistOutboundSidecar { - context: journal.context, - authenticated: locally_authenticated_with_marker(&own_candidate, 0xC2), - }) - .unwrap_err(), - JournalErrorV1::OutboundAuthenticationContextMismatch - ); - } -} diff --git a/crates/starfish-core/src/starfish_rbc_dag/mod.rs b/crates/starfish-core/src/starfish_rbc_dag/mod.rs deleted file mode 100644 index 2a3afd17..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag/mod.rs +++ /dev/null @@ -1,2516 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Canonical carrier types for the experimental embedded-RBC Starfish DAG. -//! -//! This module is deliberately independent from the implemented direct-message -//! `starfish_rbc` protocol. Runtime and consensus integration are later -//! milestones. - -pub mod journal; -pub mod model; -pub mod projection; - -use std::{ - collections::{BTreeSet, HashSet}, - error::Error, - fmt, - sync::Arc, -}; - -use crate::{ - committee::Committee, - crypto::{ - Blake3Hasher, MAC_TAG_SIZE, ML_DSA_44_SIGNATURE_SIZE, ML_DSA_65_SIGNATURE_SIZE, MacKey, - MacTag, MlDsa44SignatureBytes, MlDsa44Signer, MlDsa65SignatureBytes, MlDsa65Signer, - SIGNATURE_SIZE, SignatureBytes, Signer, TransactionsCommitment, - }, - types::{ - AuthorityIndex, BlockAuthenticationScheme, BlockDigest, BlockReference, MAX_COMMITTEE_SIZE, - RoundNumber, TimestampNs, - }, -}; - -pub const CARRIER_FORMAT_VERSION_V1: u8 = 1; -pub const CARRIER_WIRE_FORMAT_VERSION_V1: u8 = 0x81; -pub const MAX_CARRIER_CONTENT_SIZE_V1: usize = 4 * 1024 * 1024; -pub const MAX_PHASE_STATEMENTS_V1: usize = 2_048; - -const CONTENT_FORMAT_FIELD: u8 = 0x00; -const AUTHOR_FIELD: u8 = 0x01; -const CARRIER_ROUND_FIELD: u8 = 0x02; -const OWN_PREV_FIELD: u8 = 0x03; -const WEAK_PARENTS_FIELD: u8 = 0x04; -const TRANSACTIONS_COMMITMENT_FIELD: u8 = 0x05; -const ACKNOWLEDGMENTS_FIELD: u8 = 0x06; -const PHASE_BATCH_FIELD: u8 = 0x07; -const CONSENSUS_VERTEX_FIELD: u8 = 0x08; -const CREATION_TIME_FIELD: u8 = 0x09; -const CONSENSUS_ROUND_FIELD: u8 = 0x01; -const STRONG_PARENTS_FIELD: u8 = 0x02; -const DELIVERY_FRONTIER_FIELD: u8 = 0x03; -const LEADER_CHOICE_FIELD: u8 = 0x04; - -const OPTION_NONE: u8 = 0; -const OPTION_SOME: u8 = 1; -const PHASE_ECHO: u8 = 0; -const PHASE_READY: u8 = 1; -const LEADER_NONE: u8 = 0; -const LEADER_VOTE: u8 = 1; -const LEADER_NO_VOTE: u8 = 2; -const BLOCK_REFERENCE_SIZE: usize = 2 + 4 + 32; -const PROTOCOL_INSTANCE_SIZE: usize = 32; -const COMMITTEE_ID_SIZE: usize = 32; -const AUTHENTICATION_DOMAIN: &[u8; 19] = b"STARFISH_RBC_DAG_V1"; -const COMMITTEE_ID_DERIVE_CONTEXT: &str = "STARFISH_RBC_DAG_V1_COMMITTEE_ID"; -const CARRIER_AUTHENTICATION_KIND: u8 = 0; -const AUTHENTICATION_BASE_SIZE: usize = 123; -const AUTHENTICATION_MAC_SIZE: usize = AUTHENTICATION_BASE_SIZE + 2; - -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum RbcPhaseStatementV1 { - Echo { target: BlockReference }, - Ready { target: BlockReference }, -} - -impl RbcPhaseStatementV1 { - pub fn target(self) -> BlockReference { - match self { - Self::Echo { target } | Self::Ready { target } => target, - } - } - - fn code(self) -> u8 { - match self { - Self::Echo { .. } => PHASE_ECHO, - Self::Ready { .. } => PHASE_READY, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct ConsensusVertexReference { - carrier: BlockReference, - consensus_round: RoundNumber, -} - -impl ConsensusVertexReference { - pub const fn new(carrier: BlockReference, consensus_round: RoundNumber) -> Self { - Self { - carrier, - consensus_round, - } - } - - pub const fn carrier(self) -> BlockReference { - self.carrier - } - - pub const fn consensus_round(self) -> RoundNumber { - self.consensus_round - } - - pub const fn author(self) -> AuthorityIndex { - self.carrier.authority - } -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub enum LeaderChoiceV1 { - Vote { - leader: ConsensusVertexReference, - }, - NoVote { - leader_author: AuthorityIndex, - leader_round: RoundNumber, - }, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ConsensusVertexV1 { - consensus_round: RoundNumber, - strong_parents: Vec, - delivery_frontier: Vec>, - leader_choice: LeaderChoiceV1, -} - -impl ConsensusVertexV1 { - pub fn new( - consensus_round: RoundNumber, - strong_parents: Vec, - delivery_frontier: Vec>, - leader_choice: LeaderChoiceV1, - ) -> Self { - Self { - consensus_round, - strong_parents, - delivery_frontier, - leader_choice, - } - } - - pub fn consensus_round(&self) -> RoundNumber { - self.consensus_round - } - - pub fn strong_parents(&self) -> &[ConsensusVertexReference] { - &self.strong_parents - } - - pub fn delivery_frontier(&self) -> &[Option] { - &self.delivery_frontier - } - - pub fn leader_choice(&self) -> LeaderChoiceV1 { - self.leader_choice - } - - /// Validate the context-free certified-projection shape of this optional - /// vertex. Callers intentionally invoke this separately from carrier - /// candidacy: failure excludes only the optional vertex. - pub fn validate_projection_shape( - &self, - enclosing_author: AuthorityIndex, - committee: &Committee, - ) -> Result<(), RbcDagProjectionError> { - if self.consensus_round == 0 { - return Err(RbcDagProjectionError::GenesisVertexEncoded); - } - if !committee.known_authority(enclosing_author) { - return Err(RbcDagProjectionError::UnknownAuthority(enclosing_author)); - } - if self.strong_parents.len() > committee.len() { - return Err(RbcDagProjectionError::InvalidStrongParentCount( - self.strong_parents.len(), - )); - } - - let parent_round = self.consensus_round - 1; - let mut previous_authority = None; - let mut parent_stake = 0u64; - let mut includes_own_previous = false; - for parent in &self.strong_parents { - let authority = parent.author(); - if !committee.known_authority(authority) { - return Err(RbcDagProjectionError::UnknownAuthority(authority)); - } - if previous_authority.is_some_and(|previous| previous >= authority) { - return Err(RbcDagProjectionError::StrongParentsNotOrdered); - } - previous_authority = Some(authority); - if parent.consensus_round != parent_round { - return Err(RbcDagProjectionError::InvalidStrongParent(*parent)); - } - if parent_round == 0 && parent.carrier != carrier_genesis_reference(authority) { - return Err(RbcDagProjectionError::InvalidStrongParent(*parent)); - } - includes_own_previous |= authority == enclosing_author; - parent_stake = parent_stake - .checked_add( - committee - .get_stake(authority) - .ok_or(RbcDagProjectionError::UnknownAuthority(authority))?, - ) - .ok_or(RbcDagProjectionError::StakeOverflow)?; - } - if parent_stake < committee.quorum_threshold() { - return Err(RbcDagProjectionError::InvalidStrongParentThreshold); - } - if !includes_own_previous { - return Err(RbcDagProjectionError::MissingOwnStrongParent); - } - - if self.delivery_frontier.len() != committee.len() { - return Err(RbcDagProjectionError::InvalidFrontierLength { - expected: committee.len(), - actual: self.delivery_frontier.len(), - }); - } - for (authority, entry) in self.delivery_frontier.iter().enumerate() { - if let Some(reference) = entry { - if reference.authority as usize != authority || reference.round == 0 { - return Err(RbcDagProjectionError::InvalidFrontierEntry { - authority: authority as AuthorityIndex, - reference: *reference, - }); - } - } - } - - let expected_leader = committee.elect_leader(parent_round); - match self.leader_choice { - LeaderChoiceV1::Vote { leader } => { - if leader.consensus_round != parent_round - || leader.author() != expected_leader - || !self.strong_parents.contains(&leader) - { - return Err(RbcDagProjectionError::InvalidLeaderVote(leader)); - } - } - LeaderChoiceV1::NoVote { - leader_author, - leader_round, - } => { - if leader_author != expected_leader - || leader_round != parent_round - || self - .strong_parents - .iter() - .any(|parent| parent.author() == expected_leader) - { - return Err(RbcDagProjectionError::InvalidNoVote { - leader_author, - leader_round, - }); - } - } - } - Ok(()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum RbcDagProjectionError { - CommitteeMismatch, - GenesisVertexEncoded, - UnknownAuthority(AuthorityIndex), - InvalidStrongParentCount(usize), - StrongParentsNotOrdered, - InvalidStrongParent(ConsensusVertexReference), - StakeOverflow, - InvalidStrongParentThreshold, - MissingOwnStrongParent, - InvalidFrontierLength { - expected: usize, - actual: usize, - }, - InvalidFrontierEntry { - authority: AuthorityIndex, - reference: BlockReference, - }, - InvalidLeaderVote(ConsensusVertexReference), - InvalidNoVote { - leader_author: AuthorityIndex, - leader_round: RoundNumber, - }, -} - -impl fmt::Display for RbcDagProjectionError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "Starfish-RBC-DAG projection error: {self:?}") - } -} - -impl Error for RbcDagProjectionError {} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CarrierHeaderV1 { - author: AuthorityIndex, - carrier_round: RoundNumber, - own_prev: BlockReference, - weak_parents: Vec, - transactions_commitment: TransactionsCommitment, - data_acknowledgments: Vec, - phase_batch: Vec, - consensus_vertex: Option, - creation_time_ns: TimestampNs, -} - -#[derive(Clone, Debug)] -pub struct CarrierHeaderV1Args { - pub author: AuthorityIndex, - pub carrier_round: RoundNumber, - pub own_prev: BlockReference, - pub weak_parents: Vec, - pub transactions_commitment: TransactionsCommitment, - pub data_acknowledgments: Vec, - pub phase_batch: Vec, - pub consensus_vertex: Option, - pub creation_time_ns: TimestampNs, -} - -impl CarrierHeaderV1 { - fn from_args(args: CarrierHeaderV1Args) -> Self { - Self { - author: args.author, - carrier_round: args.carrier_round, - own_prev: args.own_prev, - weak_parents: args.weak_parents, - transactions_commitment: args.transactions_commitment, - data_acknowledgments: args.data_acknowledgments, - phase_batch: args.phase_batch, - consensus_vertex: args.consensus_vertex, - creation_time_ns: args.creation_time_ns, - } - } -} - -impl CarrierHeaderV1 { - pub fn author(&self) -> AuthorityIndex { - self.author - } - - pub fn carrier_round(&self) -> RoundNumber { - self.carrier_round - } - - pub fn own_prev(&self) -> BlockReference { - self.own_prev - } - - pub fn weak_parents(&self) -> &[BlockReference] { - &self.weak_parents - } - - pub fn transactions_commitment(&self) -> TransactionsCommitment { - self.transactions_commitment - } - - pub fn data_acknowledgments(&self) -> &[BlockReference] { - &self.data_acknowledgments - } - - pub fn phase_batch(&self) -> &[RbcPhaseStatementV1] { - &self.phase_batch - } - - pub fn consensus_vertex(&self) -> Option<&ConsensusVertexV1> { - self.consensus_vertex.as_ref() - } - - pub fn creation_time_ns(&self) -> TimestampNs { - self.creation_time_ns - } - - /// Exact expanded canonical content bytes committed by the carrier - /// reference. Authentication is deliberately absent. - fn canonical_content_bytes(&self) -> Result, RbcDagError> { - encode_header(self, AckEncoding::Expanded) - } - - /// Canonical wire bytes. The acknowledgment log is represented by its - /// unique maximal physical-parent suffix and the remaining exact log. - fn canonical_wire_bytes(&self) -> Result, RbcDagError> { - encode_header(self, AckEncoding::Compressed) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CandidateCarrierV1 { - header: Arc, - reference: BlockReference, - committee_id: RbcDagCommitteeId, -} - -impl CandidateCarrierV1 { - pub fn try_new(args: CarrierHeaderV1Args, committee: &Committee) -> Result { - Self::try_from_header(CarrierHeaderV1::from_args(args), committee, None) - } - - pub fn try_from_header( - mut header: CarrierHeaderV1, - committee: &Committee, - expected_reference: Option, - ) -> Result { - normalize_acknowledgments(&mut header)?; - validate_outer_header(&header, committee)?; - let reference = carrier_reference(&header)?; - let committee_id = RbcDagCommitteeId::derive(committee)?; - if let Some(expected) = expected_reference { - if expected != reference { - return Err(RbcDagError::ReferenceMismatch { - expected, - actual: reference, - }); - } - } - Ok(Self { - header: Arc::new(header), - reference, - committee_id, - }) - } - - pub fn decode_content( - bytes: &[u8], - committee: &Committee, - expected_reference: Option, - ) -> Result { - let header = decode_header(bytes, AckEncoding::Expanded)?; - let candidate = Self::try_from_header(header, committee, expected_reference)?; - if candidate.canonical_content_bytes()?.as_slice() != bytes { - return Err(RbcDagError::NonCanonicalAcknowledgments); - } - Ok(candidate) - } - - pub fn decode_wire( - bytes: &[u8], - committee: &Committee, - expected_reference: Option, - ) -> Result { - let header = decode_header(bytes, AckEncoding::Compressed)?; - Self::try_from_header(header, committee, expected_reference) - } - - pub fn header(&self) -> &CarrierHeaderV1 { - &self.header - } - - pub fn reference(&self) -> BlockReference { - self.reference - } - - pub fn committee_id(&self) -> RbcDagCommitteeId { - self.committee_id - } - - pub fn canonical_content_bytes(&self) -> Result, RbcDagError> { - self.header.canonical_content_bytes() - } - - pub fn canonical_wire_bytes(&self) -> Result, RbcDagError> { - self.header.canonical_wire_bytes() - } - - pub fn validate_consensus_vertex( - &self, - committee: &Committee, - ) -> Result, RbcDagProjectionError> { - let committee_id = RbcDagCommitteeId::derive(committee) - .map_err(|_| RbcDagProjectionError::CommitteeMismatch)?; - if committee_id != self.committee_id { - return Err(RbcDagProjectionError::CommitteeMismatch); - } - let Some(vertex) = self.header.consensus_vertex() else { - return Ok(None); - }; - vertex.validate_projection_shape(self.header.author(), committee)?; - Ok(Some(vertex)) - } -} - -#[derive(Clone, Eq, PartialEq)] -pub struct FlatMacVector { - bytes: Box<[u8]>, -} - -impl FlatMacVector { - pub fn from_tags(tags: &[MacTag]) -> Result { - if tags.len() > MAX_COMMITTEE_SIZE as usize { - return Err(RbcDagError::InvalidMacVectorLength { - expected: MAX_COMMITTEE_SIZE as usize * MAC_TAG_SIZE, - actual: tags.len().saturating_mul(MAC_TAG_SIZE), - }); - } - let mut bytes = Vec::with_capacity(tags.len() * MAC_TAG_SIZE); - for tag in tags { - bytes.extend_from_slice(tag.as_ref()); - } - Ok(Self { - bytes: bytes.into_boxed_slice(), - }) - } - - pub fn from_bytes(bytes: Vec) -> Result { - if !bytes.chunks_exact(MAC_TAG_SIZE).remainder().is_empty() - || bytes.len() > MAX_COMMITTEE_SIZE as usize * MAC_TAG_SIZE - { - return Err(RbcDagError::InvalidFlatMacVectorLength(bytes.len())); - } - Ok(Self { - bytes: bytes.into_boxed_slice(), - }) - } - - pub fn as_bytes(&self) -> &[u8] { - &self.bytes - } - - pub fn len(&self) -> usize { - self.bytes.len() / MAC_TAG_SIZE - } - - pub fn is_empty(&self) -> bool { - self.bytes.is_empty() - } - - pub fn tag(&self, authority: AuthorityIndex) -> Option { - let start = authority as usize * MAC_TAG_SIZE; - let end = start.checked_add(MAC_TAG_SIZE)?; - let mut bytes = [0; MAC_TAG_SIZE]; - bytes.copy_from_slice(self.bytes.get(start..end)?); - Some(MacTag::from_bytes(bytes)) - } -} - -impl fmt::Debug for FlatMacVector { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("FlatMacVector") - .field("tag_count", &self.len()) - .finish() - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum CarrierAuthenticationV1 { - Ed25519(SignatureBytes), - MlDsa44(MlDsa44SignatureBytes), - MlDsa65(MlDsa65SignatureBytes), - MacVector(FlatMacVector), -} - -impl CarrierAuthenticationV1 { - pub fn scheme(&self) -> BlockAuthenticationScheme { - match self { - Self::Ed25519(_) => BlockAuthenticationScheme::Ed25519, - Self::MlDsa44(_) => BlockAuthenticationScheme::MlDsa44, - Self::MlDsa65(_) => BlockAuthenticationScheme::MlDsa65, - Self::MacVector(_) => BlockAuthenticationScheme::MacVector, - } - } - - /// Versioned sidecar wire bytes. `FlatMacVector` itself remains the raw - /// concatenation of tags; the envelope supplies version and scheme. - pub fn canonical_wire_bytes(&self) -> Vec { - let mut bytes = Vec::new(); - bytes.push(CONTENT_FORMAT_FIELD); - bytes.push(CARRIER_FORMAT_VERSION_V1); - bytes.push(authentication_scheme_code(self.scheme())); - match self { - Self::Ed25519(signature) => bytes.extend_from_slice(signature.as_ref()), - Self::MlDsa44(signature) => bytes.extend_from_slice(signature.as_ref()), - Self::MlDsa65(signature) => bytes.extend_from_slice(signature.as_ref()), - Self::MacVector(vector) => bytes.extend_from_slice(vector.as_bytes()), - } - bytes - } - - pub fn decode_wire(bytes: &[u8], committee: &Committee) -> Result { - validate_committee(committee)?; - let mut decoder = Decoder::new(bytes); - decoder.expect_marker(CONTENT_FORMAT_FIELD)?; - let version = decoder.read_u8()?; - if version != CARRIER_FORMAT_VERSION_V1 { - return Err(RbcDagError::UnsupportedVersion(version)); - } - let scheme = decode_authentication_scheme(decoder.read_u8()?)?; - let authentication = match scheme { - BlockAuthenticationScheme::Ed25519 => Self::Ed25519(SignatureBytes::from_bytes( - decoder.read_array::()?, - )), - BlockAuthenticationScheme::MlDsa44 => Self::MlDsa44(MlDsa44SignatureBytes::from_bytes( - decoder.read_array::()?, - )), - BlockAuthenticationScheme::MlDsa65 => Self::MlDsa65(MlDsa65SignatureBytes::from_bytes( - decoder.read_array::()?, - )), - BlockAuthenticationScheme::MacVector => { - let expected = committee - .len() - .checked_mul(MAC_TAG_SIZE) - .ok_or(RbcDagError::InvalidCommittee("MAC vector length overflow"))?; - let vector = FlatMacVector::from_bytes(decoder.take(expected)?.to_vec())?; - Self::MacVector(vector) - } - }; - decoder.finish()?; - Ok(authentication) - } -} - -pub enum CarrierAuthorizerV1<'a> { - Ed25519 { - authority: AuthorityIndex, - signer: &'a Signer, - }, - MlDsa44 { - authority: AuthorityIndex, - signer: &'a MlDsa44Signer, - }, - MlDsa65 { - authority: AuthorityIndex, - signer: &'a MlDsa65Signer, - }, - MacVector { - authority: AuthorityIndex, - keys: &'a [MacKey], - }, -} - -impl CarrierAuthorizerV1<'_> { - fn scheme(&self) -> BlockAuthenticationScheme { - match self { - Self::Ed25519 { .. } => BlockAuthenticationScheme::Ed25519, - Self::MlDsa44 { .. } => BlockAuthenticationScheme::MlDsa44, - Self::MlDsa65 { .. } => BlockAuthenticationScheme::MlDsa65, - Self::MacVector { .. } => BlockAuthenticationScheme::MacVector, - } - } - - fn authority(&self) -> AuthorityIndex { - match self { - Self::Ed25519 { authority, .. } - | Self::MlDsa44 { authority, .. } - | Self::MlDsa65 { authority, .. } - | Self::MacVector { authority, .. } => *authority, - } - } -} - -#[derive(Clone, Copy, Eq, Hash, PartialEq)] -pub struct RbcDagProtocolInstanceId([u8; PROTOCOL_INSTANCE_SIZE]); - -impl RbcDagProtocolInstanceId { - pub fn new(bytes: [u8; PROTOCOL_INSTANCE_SIZE]) -> Result { - if bytes.iter().all(|byte| *byte == 0) { - return Err(RbcDagError::ZeroProtocolInstance); - } - Ok(Self(bytes)) - } - - pub fn as_bytes(&self) -> &[u8; PROTOCOL_INSTANCE_SIZE] { - &self.0 - } -} - -impl fmt::Debug for RbcDagProtocolInstanceId { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "RbcDagInstance({})", hex::encode(&self.0[..4])) - } -} - -#[derive(Clone, Copy, Eq, Hash, PartialEq)] -pub struct RbcDagCommitteeId([u8; COMMITTEE_ID_SIZE]); - -impl RbcDagCommitteeId { - pub fn derive(committee: &Committee) -> Result { - validate_committee(committee)?; - let committee_size = u16::try_from(committee.len()) - .map_err(|_| RbcDagError::InvalidCommittee("committee too large"))?; - let info_length = u16::try_from(committee.info_length()) - .map_err(|_| RbcDagError::InvalidCommittee("information length too large"))?; - let mut hasher = Blake3Hasher::new_derive_key(COMMITTEE_ID_DERIVE_CONTEXT); - hasher.update(&committee_size.to_be_bytes()); - hasher.update(&committee.validity_threshold().to_be_bytes()); - hasher.update(&committee.quorum_threshold().to_be_bytes()); - hasher.update(&info_length.to_be_bytes()); - hasher.update(&committee.optimistic_fast_threshold().to_be_bytes()); - hasher.update(&committee.optimistic_vote_threshold().to_be_bytes()); - hasher.update(&committee.optimistic_ready_threshold().to_be_bytes()); - for authority in committee.authorities() { - let stake = committee - .get_stake(authority) - .ok_or(RbcDagError::UnknownAuthority(authority))?; - let public_key = committee - .get_public_key(authority) - .ok_or(RbcDagError::UnknownAuthority(authority))?; - let bls_public_key = committee - .get_bls_public_key(authority) - .ok_or(RbcDagError::UnknownAuthority(authority))?; - let ml_dsa_44_public_key = committee - .get_ml_dsa_44_public_key(authority) - .ok_or(RbcDagError::UnknownAuthority(authority))?; - let ml_dsa_65_public_key = committee - .get_ml_dsa_65_public_key(authority) - .ok_or(RbcDagError::UnknownAuthority(authority))?; - hasher.update(&authority.to_be_bytes()); - hasher.update(&stake.to_be_bytes()); - hasher.update(&public_key.to_bytes()); - hasher.update(&bls_public_key.to_bytes()); - hasher.update(&ml_dsa_44_public_key.to_bytes()); - hasher.update(&ml_dsa_65_public_key.to_bytes()); - } - Ok(Self(hasher.finalize().into())) - } - - pub fn as_bytes(&self) -> &[u8; COMMITTEE_ID_SIZE] { - &self.0 - } -} - -impl fmt::Debug for RbcDagCommitteeId { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "RbcDagCommittee({})", hex::encode(&self.0[..4])) - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct RbcDagContextV1 { - protocol_instance: RbcDagProtocolInstanceId, - committee_id: RbcDagCommitteeId, - authentication_scheme: BlockAuthenticationScheme, -} - -impl RbcDagContextV1 { - pub fn new( - protocol_instance: RbcDagProtocolInstanceId, - committee: &Committee, - authentication_scheme: BlockAuthenticationScheme, - ) -> Result { - Ok(Self { - protocol_instance, - committee_id: RbcDagCommitteeId::derive(committee)?, - authentication_scheme, - }) - } - - pub fn protocol_instance(&self) -> RbcDagProtocolInstanceId { - self.protocol_instance - } - - pub fn committee_id(&self) -> RbcDagCommitteeId { - self.committee_id - } - - pub fn authentication_scheme(&self) -> BlockAuthenticationScheme { - self.authentication_scheme - } - - pub fn authenticate( - &self, - candidate: &CandidateCarrierV1, - committee: &Committee, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - self.ensure_committee(committee)?; - self.ensure_candidate(candidate)?; - if authorizer.scheme() != self.authentication_scheme { - return Err(RbcDagError::AuthenticationSchemeMismatch); - } - let reference = candidate.reference; - if authorizer.authority() != reference.authority { - return Err(RbcDagError::AuthorizerAuthorityMismatch { - expected: reference.authority, - actual: authorizer.authority(), - }); - } - match authorizer { - CarrierAuthorizerV1::Ed25519 { signer, .. } => { - let expected = committee - .get_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - if &signer.public_key() != expected { - return Err(RbcDagError::AuthorizerKeyMismatch); - } - Ok(CarrierAuthenticationV1::Ed25519(signer.sign_digest( - &self.public_authentication_digest(reference), - ))) - } - CarrierAuthorizerV1::MlDsa44 { signer, .. } => { - let expected = committee - .get_ml_dsa_44_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - if &signer.public_key() != expected { - return Err(RbcDagError::AuthorizerKeyMismatch); - } - let digest = BlockDigest::from(self.public_authentication_digest(reference)); - Ok(CarrierAuthenticationV1::MlDsa44( - signer.sign_digest(&digest), - )) - } - CarrierAuthorizerV1::MlDsa65 { signer, .. } => { - let expected = committee - .get_ml_dsa_65_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - if &signer.public_key() != expected { - return Err(RbcDagError::AuthorizerKeyMismatch); - } - let digest = BlockDigest::from(self.public_authentication_digest(reference)); - Ok(CarrierAuthenticationV1::MlDsa65( - signer.sign_digest(&digest), - )) - } - CarrierAuthorizerV1::MacVector { keys, .. } => { - if keys.len() != committee.len() { - return Err(RbcDagError::InvalidKeyringLength { - expected: committee.len(), - actual: keys.len(), - }); - } - let tags = committee - .authorities() - .map(|recipient| { - keys[recipient as usize].compute_rbc_tag( - &self.mac_authentication_statement(reference, recipient), - ) - }) - .collect::>(); - Ok(CarrierAuthenticationV1::MacVector( - FlatMacVector::from_tags(&tags)?, - )) - } - } - } - - /// Generate the exact authentication sidecar for a locally authored - /// carrier and bind it to the candidate and protocol context. - /// - /// The returned capability has private fields so persistence and network - /// adapters cannot substitute a freely constructed, same-scheme sidecar - /// for the one produced by the configured authorizer. - pub fn authenticate_local( - &self, - candidate: CandidateCarrierV1, - committee: &Committee, - authorizer: CarrierAuthorizerV1<'_>, - ) -> Result { - let authentication = self.authenticate(&candidate, committee, authorizer)?; - Ok(LocallyAuthenticatedCarrierV1 { - candidate, - authentication, - context: *self, - }) - } - - pub fn verify_authentication( - &self, - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - receiver: AuthorityIndex, - committee: &Committee, - mac_keys: &[MacKey], - ) -> Result { - self.ensure_committee(committee)?; - self.ensure_candidate(&candidate)?; - if !committee.known_authority(receiver) { - return Err(RbcDagError::UnknownAuthority(receiver)); - } - if authentication.scheme() != self.authentication_scheme { - return Err(RbcDagError::AuthenticationSchemeMismatch); - } - let reference = candidate.reference; - match &authentication { - CarrierAuthenticationV1::Ed25519(signature) => { - let public_key = committee - .get_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - public_key - .verify_digest_signature( - &self.public_authentication_digest(reference), - signature, - ) - .map_err(|_| RbcDagError::InvalidAuthentication)?; - } - CarrierAuthenticationV1::MlDsa44(signature) => { - let public_key = committee - .get_ml_dsa_44_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - let digest = BlockDigest::from(self.public_authentication_digest(reference)); - public_key - .verify_digest_signature(&digest, signature) - .map_err(|_| RbcDagError::InvalidAuthentication)?; - } - CarrierAuthenticationV1::MlDsa65(signature) => { - let public_key = committee - .get_ml_dsa_65_public_key(reference.authority) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - let digest = BlockDigest::from(self.public_authentication_digest(reference)); - public_key - .verify_digest_signature(&digest, signature) - .map_err(|_| RbcDagError::InvalidAuthentication)?; - } - CarrierAuthenticationV1::MacVector(vector) => { - let expected_length = committee.len() * MAC_TAG_SIZE; - if vector.as_bytes().len() != expected_length { - return Err(RbcDagError::InvalidMacVectorLength { - expected: expected_length, - actual: vector.as_bytes().len(), - }); - } - if mac_keys.len() != committee.len() { - return Err(RbcDagError::InvalidKeyringLength { - expected: committee.len(), - actual: mac_keys.len(), - }); - } - let key = mac_keys - .get(reference.authority as usize) - .ok_or(RbcDagError::UnknownAuthority(reference.authority))?; - let expected = - key.compute_rbc_tag(&self.mac_authentication_statement(reference, receiver)); - let actual = vector - .tag(receiver) - .ok_or(RbcDagError::InvalidAuthentication)?; - if actual != expected { - return Err(RbcDagError::InvalidAuthentication); - } - } - } - Ok(AuthenticatedCarrierV1 { - candidate, - authentication, - context: *self, - receiver, - }) - } - - pub fn public_authentication_statement( - &self, - reference: BlockReference, - ) -> [u8; AUTHENTICATION_BASE_SIZE] { - encode_authentication_base(self, reference) - } - - pub fn mac_authentication_statement( - &self, - reference: BlockReference, - recipient: AuthorityIndex, - ) -> [u8; AUTHENTICATION_MAC_SIZE] { - let mut statement = [0; AUTHENTICATION_MAC_SIZE]; - statement[..AUTHENTICATION_BASE_SIZE] - .copy_from_slice(&encode_authentication_base(self, reference)); - statement[AUTHENTICATION_BASE_SIZE..].copy_from_slice(&recipient.to_be_bytes()); - statement - } - - pub fn public_authentication_digest(&self, reference: BlockReference) -> [u8; 32] { - blake3::hash(&self.public_authentication_statement(reference)).into() - } - - fn ensure_committee(&self, committee: &Committee) -> Result<(), RbcDagError> { - let actual = RbcDagCommitteeId::derive(committee)?; - if actual != self.committee_id { - return Err(RbcDagError::CommitteeIdMismatch); - } - Ok(()) - } - - fn ensure_candidate(&self, candidate: &CandidateCarrierV1) -> Result<(), RbcDagError> { - if candidate.committee_id != self.committee_id { - return Err(RbcDagError::CandidateCommitteeMismatch); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct AuthenticatedCarrierV1 { - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - context: RbcDagContextV1, - receiver: AuthorityIndex, -} - -/// Opaque proof that the configured local authorizer generated a carrier's -/// exact sidecar. This is distinct from [`AuthenticatedCarrierV1`], which -/// proves that one receiver verified an inbound sidecar. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct LocallyAuthenticatedCarrierV1 { - candidate: CandidateCarrierV1, - authentication: CarrierAuthenticationV1, - context: RbcDagContextV1, -} - -impl LocallyAuthenticatedCarrierV1 { - pub fn candidate(&self) -> &CandidateCarrierV1 { - &self.candidate - } - - pub fn reference(&self) -> BlockReference { - self.candidate.reference() - } - - pub fn authentication(&self) -> &CarrierAuthenticationV1 { - &self.authentication - } - - pub fn context(&self) -> RbcDagContextV1 { - self.context - } - - pub fn into_parts(self) -> (CandidateCarrierV1, CarrierAuthenticationV1, RbcDagContextV1) { - (self.candidate, self.authentication, self.context) - } -} - -impl AuthenticatedCarrierV1 { - pub fn candidate(&self) -> &CandidateCarrierV1 { - &self.candidate - } - - pub fn header(&self) -> &CarrierHeaderV1 { - self.candidate.header() - } - - pub fn reference(&self) -> BlockReference { - self.candidate.reference() - } - - pub fn authentication(&self) -> &CarrierAuthenticationV1 { - &self.authentication - } - - pub fn context(&self) -> RbcDagContextV1 { - self.context - } - - pub fn receiver(&self) -> AuthorityIndex { - self.receiver - } - - pub fn into_parts( - self, - ) -> ( - CandidateCarrierV1, - CarrierAuthenticationV1, - RbcDagContextV1, - AuthorityIndex, - ) { - ( - self.candidate, - self.authentication, - self.context, - self.receiver, - ) - } -} - -fn authentication_scheme_code(authentication_scheme: BlockAuthenticationScheme) -> u8 { - match authentication_scheme { - BlockAuthenticationScheme::Ed25519 => 0, - BlockAuthenticationScheme::MlDsa44 => 1, - BlockAuthenticationScheme::MlDsa65 => 2, - BlockAuthenticationScheme::MacVector => 3, - } -} - -fn decode_authentication_scheme(code: u8) -> Result { - match code { - 0 => Ok(BlockAuthenticationScheme::Ed25519), - 1 => Ok(BlockAuthenticationScheme::MlDsa44), - 2 => Ok(BlockAuthenticationScheme::MlDsa65), - 3 => Ok(BlockAuthenticationScheme::MacVector), - other => Err(RbcDagError::InvalidAuthenticationScheme(other)), - } -} - -fn encode_authentication_base( - context: &RbcDagContextV1, - reference: BlockReference, -) -> [u8; AUTHENTICATION_BASE_SIZE] { - let mut statement = [0; AUTHENTICATION_BASE_SIZE]; - statement[..19].copy_from_slice(AUTHENTICATION_DOMAIN); - statement[19] = CARRIER_AUTHENTICATION_KIND; - statement[20] = authentication_scheme_code(context.authentication_scheme); - statement[21..53].copy_from_slice(&context.protocol_instance.0); - statement[53..85].copy_from_slice(&context.committee_id.0); - statement[85..87].copy_from_slice(&reference.authority.to_be_bytes()); - statement[87..91].copy_from_slice(&reference.round.to_be_bytes()); - statement[91..123].copy_from_slice(reference.digest.as_ref()); - statement -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum RbcDagError { - UnexpectedEnd, - TrailingBytes(usize), - UnsupportedVersion(u8), - InvalidMarker { - expected: u8, - actual: u8, - }, - InvalidOption(u8), - InvalidPhase(u8), - InvalidLeaderChoice(u8), - VectorTooLong { - field: &'static str, - count: usize, - }, - ContentTooLarge(usize), - NonCanonicalAcknowledgments, - InvalidCommittee(&'static str), - UnknownAuthority(AuthorityIndex), - GenesisCarrier, - InvalidOwnPrevious, - InvalidWeakParent(BlockReference), - WeakParentsNotOrdered, - InvalidCarrierThreshold, - InvalidAcknowledgment(BlockReference), - DuplicateAcknowledgment(BlockReference), - InvalidPhaseTarget(BlockReference), - DuplicatePhaseStatement(RbcPhaseStatementV1), - InvalidConsensusRound, - ZeroProtocolInstance, - CommitteeIdMismatch, - CandidateCommitteeMismatch, - AuthenticationSchemeMismatch, - InvalidAuthenticationScheme(u8), - AuthorizerAuthorityMismatch { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - AuthorizerKeyMismatch, - InvalidAuthentication, - InvalidFlatMacVectorLength(usize), - InvalidMacVectorLength { - expected: usize, - actual: usize, - }, - InvalidKeyringLength { - expected: usize, - actual: usize, - }, - ReferenceMismatch { - expected: BlockReference, - actual: BlockReference, - }, -} - -impl fmt::Display for RbcDagError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "Starfish-RBC-DAG error: {self:?}") - } -} - -impl Error for RbcDagError {} - -fn validate_outer_header( - header: &CarrierHeaderV1, - committee: &Committee, -) -> Result<(), RbcDagError> { - validate_committee(committee)?; - if header.carrier_round == 0 { - return Err(RbcDagError::GenesisCarrier); - } - if !committee.known_authority(header.author) { - return Err(RbcDagError::UnknownAuthority(header.author)); - } - if header.own_prev.authority != header.author - || header.own_prev.round.checked_add(1) != Some(header.carrier_round) - || (header.carrier_round == 1 - && header.own_prev != carrier_genesis_reference(header.author)) - { - return Err(RbcDagError::InvalidOwnPrevious); - } - - if header.weak_parents.len() >= committee.len() - || header.weak_parents.len() > MAX_COMMITTEE_SIZE as usize - 1 - { - return Err(RbcDagError::VectorTooLong { - field: "weak parents", - count: header.weak_parents.len(), - }); - } - let mut previous_authority = None; - let mut parent_stake = committee - .get_stake(header.author) - .ok_or(RbcDagError::UnknownAuthority(header.author))?; - for parent in &header.weak_parents { - if !committee.known_authority(parent.authority) { - return Err(RbcDagError::UnknownAuthority(parent.authority)); - } - if parent.authority == header.author - || parent.round.checked_add(1) != Some(header.carrier_round) - || (header.carrier_round == 1 && *parent != carrier_genesis_reference(parent.authority)) - { - return Err(RbcDagError::InvalidWeakParent(*parent)); - } - if previous_authority.is_some_and(|previous| previous >= parent.authority) { - return Err(RbcDagError::WeakParentsNotOrdered); - } - previous_authority = Some(parent.authority); - parent_stake = parent_stake - .checked_add( - committee - .get_stake(parent.authority) - .ok_or(RbcDagError::UnknownAuthority(parent.authority))?, - ) - .ok_or(RbcDagError::InvalidCommittee("stake overflow"))?; - } - if parent_stake < committee.quorum_threshold() { - return Err(RbcDagError::InvalidCarrierThreshold); - } - - if header.data_acknowledgments.len() > u16::MAX as usize { - return Err(RbcDagError::VectorTooLong { - field: "acknowledgments", - count: header.data_acknowledgments.len(), - }); - } - let mut acknowledgments = BTreeSet::new(); - for acknowledgment in &header.data_acknowledgments { - if !committee.known_authority(acknowledgment.authority) - || acknowledgment.round == 0 - || acknowledgment.round > header.carrier_round - { - return Err(RbcDagError::InvalidAcknowledgment(*acknowledgment)); - } - if !acknowledgments.insert(*acknowledgment) { - return Err(RbcDagError::DuplicateAcknowledgment(*acknowledgment)); - } - } - - let phase_limit = usize::min(MAX_PHASE_STATEMENTS_V1, committee.len().saturating_mul(4)); - if header.phase_batch.len() > phase_limit { - return Err(RbcDagError::VectorTooLong { - field: "phase statements", - count: header.phase_batch.len(), - }); - } - let mut phase_statements = HashSet::new(); - for statement in &header.phase_batch { - let target = statement.target(); - if !committee.known_authority(target.authority) - || target.round == 0 - || target.round >= header.carrier_round - { - return Err(RbcDagError::InvalidPhaseTarget(target)); - } - if !phase_statements.insert((statement.code(), target.authority, target.round)) { - return Err(RbcDagError::DuplicatePhaseStatement(*statement)); - } - } - - if let Some(vertex) = &header.consensus_vertex { - if vertex.consensus_round == 0 { - return Err(RbcDagError::InvalidConsensusRound); - } - for (field, count) in [ - ("strong parents", vertex.strong_parents.len()), - ("delivery frontier", vertex.delivery_frontier.len()), - ] { - if count > MAX_COMMITTEE_SIZE as usize { - return Err(RbcDagError::VectorTooLong { field, count }); - } - } - } - - let content_size = header.canonical_content_bytes()?.len(); - if content_size > MAX_CARRIER_CONTENT_SIZE_V1 { - return Err(RbcDagError::ContentTooLarge(content_size)); - } - // Candidacy guarantees that the same logical carrier has a canonical - // transport representation as well as an identity representation. - header.canonical_wire_bytes()?; - Ok(()) -} - -fn carrier_reference(header: &CarrierHeaderV1) -> Result { - let bytes = header.canonical_content_bytes()?; - Ok(BlockReference { - round: header.carrier_round, - authority: header.author, - digest: BlockDigest::from(*blake3::hash(&bytes).as_bytes()), - }) -} - -/// Fixed virtual carrier-genesis reference for one authority. -pub fn carrier_genesis_reference(authority: AuthorityIndex) -> BlockReference { - let digest = BlockDigest::new_without_transactions(authority, 0, &[], &[], 0, None, None); - BlockReference { - round: 0, - authority, - digest, - } -} - -#[derive(Clone, Copy)] -enum AckEncoding { - Expanded, - Compressed, -} - -fn encode_header( - header: &CarrierHeaderV1, - acknowledgment_encoding: AckEncoding, -) -> Result, RbcDagError> { - let mut bytes = Vec::new(); - bytes.push(CONTENT_FORMAT_FIELD); - bytes.push(match acknowledgment_encoding { - AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, - AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, - }); - bytes.push(AUTHOR_FIELD); - bytes.extend_from_slice(&header.author.to_be_bytes()); - bytes.push(CARRIER_ROUND_FIELD); - bytes.extend_from_slice(&header.carrier_round.to_be_bytes()); - bytes.push(OWN_PREV_FIELD); - encode_reference(&mut bytes, header.own_prev); - bytes.push(WEAK_PARENTS_FIELD); - encode_count(&mut bytes, "weak parents", header.weak_parents.len())?; - for parent in &header.weak_parents { - encode_reference(&mut bytes, *parent); - } - bytes.push(TRANSACTIONS_COMMITMENT_FIELD); - bytes.extend_from_slice(header.transactions_commitment.as_ref()); - bytes.push(ACKNOWLEDGMENTS_FIELD); - match acknowledgment_encoding { - AckEncoding::Expanded => { - encode_count( - &mut bytes, - "acknowledgments", - header.data_acknowledgments.len(), - )?; - for acknowledgment in &header.data_acknowledgments { - encode_reference(&mut bytes, *acknowledgment); - } - } - AckEncoding::Compressed => { - let (intersection, extras) = compressed_acknowledgments(header)?; - bytes.extend_from_slice(&intersection.to_be_bytes()); - encode_count(&mut bytes, "extra acknowledgments", extras.len())?; - for acknowledgment in &extras { - encode_reference(&mut bytes, *acknowledgment); - } - } - } - bytes.push(PHASE_BATCH_FIELD); - encode_count(&mut bytes, "phase statements", header.phase_batch.len())?; - for statement in &header.phase_batch { - bytes.push(statement.code()); - encode_reference(&mut bytes, statement.target()); - } - bytes.push(CONSENSUS_VERTEX_FIELD); - match &header.consensus_vertex { - None => bytes.push(OPTION_NONE), - Some(vertex) => { - bytes.push(OPTION_SOME); - encode_consensus_vertex(&mut bytes, vertex)?; - } - } - bytes.push(CREATION_TIME_FIELD); - bytes.extend_from_slice(&header.creation_time_ns.to_be_bytes()); - if bytes.len() > MAX_CARRIER_CONTENT_SIZE_V1 { - return Err(RbcDagError::ContentTooLarge(bytes.len())); - } - Ok(bytes) -} - -fn encode_consensus_vertex( - bytes: &mut Vec, - vertex: &ConsensusVertexV1, -) -> Result<(), RbcDagError> { - bytes.push(CONSENSUS_ROUND_FIELD); - bytes.extend_from_slice(&vertex.consensus_round.to_be_bytes()); - bytes.push(STRONG_PARENTS_FIELD); - encode_count(bytes, "strong parents", vertex.strong_parents.len())?; - for parent in &vertex.strong_parents { - encode_consensus_reference(bytes, *parent); - } - bytes.push(DELIVERY_FRONTIER_FIELD); - encode_count(bytes, "delivery frontier", vertex.delivery_frontier.len())?; - for entry in &vertex.delivery_frontier { - match entry { - None => bytes.push(OPTION_NONE), - Some(reference) => { - bytes.push(OPTION_SOME); - encode_reference(bytes, *reference); - } - } - } - bytes.push(LEADER_CHOICE_FIELD); - match vertex.leader_choice { - LeaderChoiceV1::Vote { leader } => { - bytes.push(LEADER_VOTE); - encode_consensus_reference(bytes, leader); - } - LeaderChoiceV1::NoVote { - leader_author, - leader_round, - } => { - bytes.push(LEADER_NO_VOTE); - bytes.extend_from_slice(&leader_author.to_be_bytes()); - bytes.extend_from_slice(&leader_round.to_be_bytes()); - } - } - Ok(()) -} - -fn encode_count(bytes: &mut Vec, field: &'static str, count: usize) -> Result<(), RbcDagError> { - let count = u16::try_from(count).map_err(|_| RbcDagError::VectorTooLong { field, count })?; - bytes.extend_from_slice(&count.to_be_bytes()); - Ok(()) -} - -fn encode_reference(bytes: &mut Vec, reference: BlockReference) { - bytes.extend_from_slice(&reference.authority.to_be_bytes()); - bytes.extend_from_slice(&reference.round.to_be_bytes()); - bytes.extend_from_slice(reference.digest.as_ref()); -} - -fn encode_consensus_reference(bytes: &mut Vec, reference: ConsensusVertexReference) { - encode_reference(bytes, reference.carrier); - bytes.extend_from_slice(&reference.consensus_round.to_be_bytes()); -} - -fn physical_parents(header: &CarrierHeaderV1) -> Vec { - let mut parents = Vec::with_capacity(header.weak_parents.len() + 1); - parents.push(header.own_prev); - parents.extend_from_slice(&header.weak_parents); - parents -} - -fn compressed_acknowledgments( - header: &CarrierHeaderV1, -) -> Result<(u16, Vec), RbcDagError> { - let parents = physical_parents(header); - let acknowledged: BTreeSet<_> = header.data_acknowledgments.iter().copied().collect(); - let mut intersection = parents.len(); - while intersection > 0 && acknowledged.contains(&parents[intersection - 1]) { - intersection -= 1; - } - let shared: BTreeSet<_> = parents[intersection..].iter().copied().collect(); - let extras = header - .data_acknowledgments - .iter() - .copied() - .filter(|reference| !shared.contains(reference)) - .collect(); - let intersection = u16::try_from(intersection).map_err(|_| RbcDagError::VectorTooLong { - field: "physical parents", - count: parents.len(), - })?; - Ok((intersection, extras)) -} - -fn normalize_acknowledgments(header: &mut CarrierHeaderV1) -> Result<(), RbcDagError> { - let mut seen = BTreeSet::new(); - for acknowledgment in &header.data_acknowledgments { - if !seen.insert(*acknowledgment) { - return Err(RbcDagError::DuplicateAcknowledgment(*acknowledgment)); - } - } - let parents = physical_parents(header); - let (intersection, extras) = compressed_acknowledgments(header)?; - let mut normalized = parents[intersection as usize..].to_vec(); - normalized.extend(extras); - header.data_acknowledgments = normalized; - Ok(()) -} - -fn decode_header( - bytes: &[u8], - acknowledgment_encoding: AckEncoding, -) -> Result { - if bytes.len() > MAX_CARRIER_CONTENT_SIZE_V1 { - return Err(RbcDagError::ContentTooLarge(bytes.len())); - } - let mut decoder = Decoder::new(bytes); - decoder.expect_marker(CONTENT_FORMAT_FIELD)?; - let version = decoder.read_u8()?; - let expected_version = match acknowledgment_encoding { - AckEncoding::Expanded => CARRIER_FORMAT_VERSION_V1, - AckEncoding::Compressed => CARRIER_WIRE_FORMAT_VERSION_V1, - }; - if version != expected_version { - return Err(RbcDagError::UnsupportedVersion(version)); - } - decoder.expect_marker(AUTHOR_FIELD)?; - let author = decoder.read_u16()?; - decoder.expect_marker(CARRIER_ROUND_FIELD)?; - let carrier_round = decoder.read_u32()?; - decoder.expect_marker(OWN_PREV_FIELD)?; - let own_prev = decoder.read_reference()?; - decoder.expect_marker(WEAK_PARENTS_FIELD)?; - let weak_count = decoder.read_count("weak parents", MAX_COMMITTEE_SIZE as usize - 1)?; - let weak_parents = decoder.read_references(weak_count)?; - decoder.expect_marker(TRANSACTIONS_COMMITMENT_FIELD)?; - let transactions_commitment = TransactionsCommitment::from_bytes(decoder.read_array()?); - decoder.expect_marker(ACKNOWLEDGMENTS_FIELD)?; - let data_acknowledgments = match acknowledgment_encoding { - AckEncoding::Expanded => { - let count = decoder.read_count("acknowledgments", u16::MAX as usize)?; - decoder.read_references(count)? - } - AckEncoding::Compressed => { - let intersection = decoder.read_u16()? as usize; - let extra_count = decoder.read_count("extra acknowledgments", u16::MAX as usize)?; - let extras = decoder.read_references(extra_count)?; - let mut parents = Vec::with_capacity(weak_parents.len() + 1); - parents.push(own_prev); - parents.extend_from_slice(&weak_parents); - if intersection > parents.len() { - return Err(RbcDagError::NonCanonicalAcknowledgments); - } - let mut acknowledgments = parents[intersection..].to_vec(); - acknowledgments.extend_from_slice(&extras); - let provisional = CarrierHeaderV1 { - author, - carrier_round, - own_prev, - weak_parents: weak_parents.clone(), - transactions_commitment, - data_acknowledgments: acknowledgments.clone(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: 0, - }; - let (canonical_intersection, canonical_extras) = - compressed_acknowledgments(&provisional)?; - if canonical_intersection as usize != intersection || canonical_extras != extras { - return Err(RbcDagError::NonCanonicalAcknowledgments); - } - acknowledgments - } - }; - decoder.expect_marker(PHASE_BATCH_FIELD)?; - let phase_count = decoder.read_count("phase statements", MAX_PHASE_STATEMENTS_V1)?; - let mut phase_batch = Vec::with_capacity(phase_count); - for _ in 0..phase_count { - let code = decoder.read_u8()?; - let target = decoder.read_reference()?; - phase_batch.push(match code { - PHASE_ECHO => RbcPhaseStatementV1::Echo { target }, - PHASE_READY => RbcPhaseStatementV1::Ready { target }, - other => return Err(RbcDagError::InvalidPhase(other)), - }); - } - decoder.expect_marker(CONSENSUS_VERTEX_FIELD)?; - let consensus_vertex = match decoder.read_u8()? { - OPTION_NONE => None, - OPTION_SOME => Some(decoder.read_consensus_vertex()?), - other => return Err(RbcDagError::InvalidOption(other)), - }; - decoder.expect_marker(CREATION_TIME_FIELD)?; - let creation_time_ns = decoder.read_u64()?; - decoder.finish()?; - Ok(CarrierHeaderV1 { - author, - carrier_round, - own_prev, - weak_parents, - transactions_commitment, - data_acknowledgments, - phase_batch, - consensus_vertex, - creation_time_ns, - }) -} - -struct Decoder<'a> { - bytes: &'a [u8], - position: usize, -} - -impl<'a> Decoder<'a> { - fn new(bytes: &'a [u8]) -> Self { - Self { bytes, position: 0 } - } - - fn take(&mut self, length: usize) -> Result<&'a [u8], RbcDagError> { - let end = self - .position - .checked_add(length) - .ok_or(RbcDagError::UnexpectedEnd)?; - let value = self - .bytes - .get(self.position..end) - .ok_or(RbcDagError::UnexpectedEnd)?; - self.position = end; - Ok(value) - } - - fn read_array(&mut self) -> Result<[u8; N], RbcDagError> { - let mut value = [0; N]; - value.copy_from_slice(self.take(N)?); - Ok(value) - } - - fn read_u8(&mut self) -> Result { - Ok(self.take(1)?[0]) - } - - fn read_u16(&mut self) -> Result { - Ok(u16::from_be_bytes(self.read_array()?)) - } - - fn read_u32(&mut self) -> Result { - Ok(u32::from_be_bytes(self.read_array()?)) - } - - fn read_u64(&mut self) -> Result { - Ok(u64::from_be_bytes(self.read_array()?)) - } - - fn expect_marker(&mut self, expected: u8) -> Result<(), RbcDagError> { - let actual = self.read_u8()?; - if actual != expected { - return Err(RbcDagError::InvalidMarker { expected, actual }); - } - Ok(()) - } - - fn read_count(&mut self, field: &'static str, maximum: usize) -> Result { - let count = self.read_u16()? as usize; - if count > maximum { - return Err(RbcDagError::VectorTooLong { field, count }); - } - Ok(count) - } - - fn read_reference(&mut self) -> Result { - let authority = self.read_u16()?; - let round = self.read_u32()?; - let digest = BlockDigest::from(self.read_array()?); - Ok(BlockReference { - round, - authority, - digest, - }) - } - - fn read_references(&mut self, count: usize) -> Result, RbcDagError> { - let byte_count = count - .checked_mul(BLOCK_REFERENCE_SIZE) - .ok_or(RbcDagError::UnexpectedEnd)?; - if self.bytes.len().saturating_sub(self.position) < byte_count { - return Err(RbcDagError::UnexpectedEnd); - } - let mut references = Vec::with_capacity(count); - for _ in 0..count { - references.push(self.read_reference()?); - } - Ok(references) - } - - fn read_consensus_reference(&mut self) -> Result { - Ok(ConsensusVertexReference::new( - self.read_reference()?, - self.read_u32()?, - )) - } - - fn read_consensus_vertex(&mut self) -> Result { - self.expect_marker(CONSENSUS_ROUND_FIELD)?; - let consensus_round = self.read_u32()?; - self.expect_marker(STRONG_PARENTS_FIELD)?; - let parent_count = self.read_count("strong parents", MAX_COMMITTEE_SIZE as usize)?; - let mut strong_parents = Vec::with_capacity(parent_count); - for _ in 0..parent_count { - strong_parents.push(self.read_consensus_reference()?); - } - self.expect_marker(DELIVERY_FRONTIER_FIELD)?; - let frontier_count = self.read_count("delivery frontier", MAX_COMMITTEE_SIZE as usize)?; - let mut delivery_frontier = Vec::with_capacity(frontier_count); - for _ in 0..frontier_count { - delivery_frontier.push(match self.read_u8()? { - OPTION_NONE => None, - OPTION_SOME => Some(self.read_reference()?), - other => return Err(RbcDagError::InvalidOption(other)), - }); - } - self.expect_marker(LEADER_CHOICE_FIELD)?; - let leader_choice = match self.read_u8()? { - LEADER_NONE => return Err(RbcDagError::InvalidLeaderChoice(LEADER_NONE)), - LEADER_VOTE => LeaderChoiceV1::Vote { - leader: self.read_consensus_reference()?, - }, - LEADER_NO_VOTE => LeaderChoiceV1::NoVote { - leader_author: self.read_u16()?, - leader_round: self.read_u32()?, - }, - other => return Err(RbcDagError::InvalidLeaderChoice(other)), - }; - Ok(ConsensusVertexV1::new( - consensus_round, - strong_parents, - delivery_frontier, - leader_choice, - )) - } - - fn finish(self) -> Result<(), RbcDagError> { - if self.position != self.bytes.len() { - return Err(RbcDagError::TrailingBytes(self.bytes.len() - self.position)); - } - Ok(()) - } -} - -fn validate_committee(committee: &Committee) -> Result<(), RbcDagError> { - if committee.is_empty() || committee.len() > MAX_COMMITTEE_SIZE as usize { - return Err(RbcDagError::InvalidCommittee("invalid committee size")); - } - let mut total_stake = 0u64; - for authority in committee.authorities() { - let stake = committee - .get_stake(authority) - .ok_or(RbcDagError::UnknownAuthority(authority))?; - if stake == 0 { - return Err(RbcDagError::InvalidCommittee("zero stake")); - } - total_stake = total_stake - .checked_add(stake) - .ok_or(RbcDagError::InvalidCommittee("stake overflow"))?; - } - let expected_validity = total_stake / 3 + 1; - let expected_quorum = total_stake - .checked_mul(2) - .ok_or(RbcDagError::InvalidCommittee("stake overflow"))? - / 3 - + 1; - if committee.validity_threshold() != expected_validity - || committee.quorum_threshold() != expected_quorum - { - return Err(RbcDagError::InvalidCommittee("threshold mismatch")); - } - let committee_size = committee.len(); - let fault_count = (committee_size - 1) / 3; - let expected_info_length = match committee_size % 3 { - 0 => fault_count + 3, - 1 => fault_count + 1, - _ => fault_count + 2, - }; - if committee.info_length() != expected_info_length { - return Err(RbcDagError::InvalidCommittee("information length mismatch")); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::crypto::{ - dummy_ml_dsa_44_signer, dummy_ml_dsa_65_signer, dummy_signer, mac_keyrings_for_test, - }; - - fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { - BlockReference { - authority, - round, - digest: BlockDigest::from([marker; 32]), - } - } - - fn quorum_others(committee: &Committee, author: AuthorityIndex) -> Vec { - let mut stake = committee.get_stake(author).unwrap(); - let mut others = Vec::new(); - for authority in committee.authorities().filter(|other| *other != author) { - if stake >= committee.quorum_threshold() { - break; - } - stake += committee.get_stake(authority).unwrap(); - others.push(authority); - } - others.sort_unstable(); - others - } - - fn args( - committee: &Committee, - author: AuthorityIndex, - carrier_round: RoundNumber, - ) -> CarrierHeaderV1Args { - let previous = carrier_round - 1; - let parent = |authority| { - if previous == 0 { - carrier_genesis_reference(authority) - } else { - reference(authority, previous, 0x40 + authority as u8) - } - }; - CarrierHeaderV1Args { - author, - carrier_round, - own_prev: parent(author), - weak_parents: quorum_others(committee, author) - .into_iter() - .map(parent) - .collect(), - transactions_commitment: TransactionsCommitment::from_bytes([0x55; 32]), - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: None, - creation_time_ns: 0x0102_0304_0506_0708, - } - } - - fn full_args(committee: &Committee) -> CarrierHeaderV1Args { - let mut args = args(committee, 3, 2); - let phase_target = reference(2, 1, 0x72); - args.data_acknowledgments = vec![*args.weak_parents.last().unwrap(), reference(2, 1, 0x61)]; - args.phase_batch = vec![ - RbcPhaseStatementV1::Echo { - target: phase_target, - }, - RbcPhaseStatementV1::Ready { - target: phase_target, - }, - ]; - let strong_parents = [0, 1, 3] - .into_iter() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - args.consensus_vertex = Some(ConsensusVertexV1::new( - 1, - strong_parents, - vec![None; committee.len()], - LeaderChoiceV1::Vote { - leader: ConsensusVertexReference::new(carrier_genesis_reference(0), 0), - }, - )); - args - } - - fn full_candidate(committee: &Committee) -> CandidateCarrierV1 { - CandidateCarrierV1::try_new(full_args(committee), committee).unwrap() - } - - #[test] - fn canonical_content_and_reference_have_frozen_golden_bytes() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - let bytes = candidate.canonical_content_bytes().unwrap(); - assert_eq!( - bytes[0..2], - [CONTENT_FORMAT_FIELD, CARRIER_FORMAT_VERSION_V1] - ); - assert_eq!( - candidate.reference().digest.as_ref(), - blake3::hash(&bytes).as_bytes() - ); - let mut expected = vec![0x00, 0x01, 0x01]; - expected.extend_from_slice(&3u16.to_be_bytes()); - expected.push(0x02); - expected.extend_from_slice(&2u32.to_be_bytes()); - expected.push(0x03); - encode_reference(&mut expected, reference(3, 1, 0x43)); - expected.extend_from_slice(&[0x04, 0x00, 0x02]); - encode_reference(&mut expected, reference(0, 1, 0x40)); - encode_reference(&mut expected, reference(1, 1, 0x41)); - expected.push(0x05); - expected.extend_from_slice(&[0x55; 32]); - expected.extend_from_slice(&[0x06, 0x00, 0x02]); - encode_reference(&mut expected, reference(1, 1, 0x41)); - encode_reference(&mut expected, reference(2, 1, 0x61)); - expected.extend_from_slice(&[0x07, 0x00, 0x02]); - for phase in [0x00, 0x01] { - expected.push(phase); - encode_reference(&mut expected, reference(2, 1, 0x72)); - } - expected.extend_from_slice(&[0x08, 0x01, 0x01]); - expected.extend_from_slice(&1u32.to_be_bytes()); - expected.extend_from_slice(&[0x02, 0x00, 0x03]); - for authority in [0, 1, 3] { - encode_reference(&mut expected, carrier_genesis_reference(authority)); - expected.extend_from_slice(&0u32.to_be_bytes()); - } - expected.extend_from_slice(&[0x03, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00]); - expected.extend_from_slice(&[0x04, 0x01]); - encode_reference(&mut expected, carrier_genesis_reference(0)); - expected.extend_from_slice(&0u32.to_be_bytes()); - expected.push(0x09); - expected.extend_from_slice(&0x0102_0304_0506_0708u64.to_be_bytes()); - assert_eq!(bytes, expected); - assert_eq!( - hex::encode(candidate.reference().digest.as_ref()), - "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f" - ); - } - - #[test] - fn every_canonical_carrier_field_is_bound_to_the_reference() { - let committee = Committee::new_test(vec![1; 4]); - let mut base = full_args(&committee); - base.data_acknowledgments.push(reference(0, 1, 0x62)); - let base_candidate = CandidateCarrierV1::try_new(base.clone(), &committee).unwrap(); - let base_reference = base_candidate.reference(); - let mut mutations = Vec::new(); - - let mut changed = base.clone(); - changed.author = 2; - changed.own_prev = reference(2, 1, 0x42); - mutations.push(("author", changed)); - - let mut changed = base.clone(); - changed.carrier_round = 3; - changed.own_prev = reference(3, 2, 0x43); - changed.weak_parents = vec![reference(0, 2, 0x40), reference(1, 2, 0x41)]; - mutations.push(("carrier round", changed)); - - let mut changed = base.clone(); - changed.own_prev.digest = BlockDigest::from([0x91; 32]); - mutations.push(("own predecessor", changed)); - - let mut changed = base.clone(); - changed.weak_parents[0].digest = BlockDigest::from([0x92; 32]); - mutations.push(("weak parent", changed)); - - let mut changed = base.clone(); - changed.weak_parents.push(reference(2, 1, 0x42)); - mutations.push(("weak parent count", changed)); - - let mut changed = base.clone(); - changed.transactions_commitment = TransactionsCommitment::from_bytes([0x93; 32]); - mutations.push(("transaction commitment", changed)); - - let mut changed = base.clone(); - changed.data_acknowledgments[1].digest = BlockDigest::from([0x94; 32]); - mutations.push(("acknowledgment", changed)); - - let mut changed = base.clone(); - changed.data_acknowledgments.push(reference(3, 1, 0x95)); - mutations.push(("acknowledgment count", changed)); - - let mut changed = base.clone(); - changed.data_acknowledgments.swap(1, 2); - mutations.push(("acknowledgment order", changed)); - - let mut changed = base.clone(); - changed.phase_batch[0] = RbcPhaseStatementV1::Ready { - target: reference(1, 1, 0x96), - }; - mutations.push(("phase kind", changed)); - - let mut changed = base.clone(); - changed.phase_batch[0] = RbcPhaseStatementV1::Echo { - target: reference(2, 1, 0x97), - }; - mutations.push(("phase target", changed)); - - let mut changed = base.clone(); - changed.phase_batch.swap(0, 1); - mutations.push(("phase order", changed)); - - let mut changed = base.clone(); - changed.phase_batch.pop(); - mutations.push(("phase count", changed)); - - let mut changed = base.clone(); - changed.consensus_vertex = None; - mutations.push(("consensus presence", changed)); - - let mut changed = base.clone(); - changed.consensus_vertex.as_mut().unwrap().consensus_round = 2; - mutations.push(("consensus round", changed)); - - let mut changed = base.clone(); - changed.consensus_vertex.as_mut().unwrap().strong_parents[0] - .carrier - .digest = BlockDigest::from([0x98; 32]); - mutations.push(("strong parent carrier", changed)); - - let mut changed = base.clone(); - changed.consensus_vertex.as_mut().unwrap().strong_parents[0].consensus_round = 1; - mutations.push(("strong parent consensus round", changed)); - - let mut changed = base.clone(); - changed - .consensus_vertex - .as_mut() - .unwrap() - .strong_parents - .insert( - 2, - ConsensusVertexReference::new(carrier_genesis_reference(2), 0), - ); - mutations.push(("strong parent count", changed)); - - let mut changed = base.clone(); - changed.consensus_vertex.as_mut().unwrap().delivery_frontier[0] = - Some(reference(0, 1, 0x99)); - mutations.push(("frontier entry", changed)); - - let mut changed = base.clone(); - changed - .consensus_vertex - .as_mut() - .unwrap() - .delivery_frontier - .pop(); - mutations.push(("frontier count", changed)); - - let mut changed = base.clone(); - changed.consensus_vertex.as_mut().unwrap().leader_choice = LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }; - mutations.push(("leader choice", changed)); - - let mut changed = base; - changed.creation_time_ns ^= 1; - mutations.push(("creation time", changed)); - - for (field, changed) in mutations { - let candidate = CandidateCarrierV1::try_new(changed, &committee) - .unwrap_or_else(|error| panic!("{field} mutation must remain encodable: {error}")); - assert_ne!(candidate.reference(), base_reference, "unbound {field}"); - let bytes = candidate.canonical_content_bytes().unwrap(); - assert!(matches!( - CandidateCarrierV1::decode_content(&bytes, &committee, Some(base_reference)), - Err(RbcDagError::ReferenceMismatch { .. }) - )); - } - } - - #[test] - fn content_and_compressed_wire_round_trip_to_same_reference() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - let content = candidate.canonical_content_bytes().unwrap(); - let wire = candidate.canonical_wire_bytes().unwrap(); - assert_eq!(content[1], CARRIER_FORMAT_VERSION_V1); - assert_eq!(wire[1], CARRIER_WIRE_FORMAT_VERSION_V1); - assert!(wire.len() < content.len()); - assert_eq!( - CandidateCarrierV1::decode_content(&content, &committee, Some(candidate.reference())) - .unwrap(), - candidate - ); - assert!(matches!( - CandidateCarrierV1::decode_content(&wire, &committee, None), - Err(RbcDagError::UnsupportedVersion( - CARRIER_WIRE_FORMAT_VERSION_V1 - )) - )); - assert!(matches!( - CandidateCarrierV1::decode_wire(&content, &committee, None), - Err(RbcDagError::UnsupportedVersion(CARRIER_FORMAT_VERSION_V1)) - )); - assert_eq!( - CandidateCarrierV1::decode_wire(&wire, &committee, Some(candidate.reference())) - .unwrap(), - candidate - ); - for end in 0..content.len() { - assert!(CandidateCarrierV1::decode_content(&content[..end], &committee, None).is_err()); - } - for end in 0..wire.len() { - assert!(CandidateCarrierV1::decode_wire(&wire[..end], &committee, None).is_err()); - } - let mut trailing = wire; - trailing.push(0); - assert!(matches!( - CandidateCarrierV1::decode_wire(&trailing, &committee, None), - Err(RbcDagError::TrailingBytes(1)) - )); - } - - #[test] - fn acknowledgment_compression_normalizes_and_rejects_duplicates() { - let committee = Committee::new_test(vec![1; 4]); - let base = args(&committee, 3, 2); - let shared = *base.weak_parents.last().unwrap(); - let extra = reference(2, 1, 0x91); - - let mut reordered = base.clone(); - reordered.data_acknowledgments = vec![extra, shared]; - let candidate = CandidateCarrierV1::try_new(reordered, &committee).unwrap(); - assert_eq!(candidate.header().data_acknowledgments(), &[shared, extra]); - - for acknowledgments in [vec![shared, shared], vec![extra, shared, shared]] { - let mut duplicate = base.clone(); - duplicate.data_acknowledgments = acknowledgments; - assert!(matches!( - CandidateCarrierV1::try_new(duplicate, &committee), - Err(RbcDagError::DuplicateAcknowledgment(reference)) if reference == shared - )); - } - } - - #[test] - fn acknowledgment_validation_handles_many_shared_hash_prefixes() { - let committee = Committee::new_test(vec![1; 4]); - let mut carrier = args(&committee, 3, 2); - carrier.data_acknowledgments = (0..8_192u64) - .map(|counter| { - let mut digest = [0xA5; 32]; - digest[24..].copy_from_slice(&counter.to_be_bytes()); - BlockReference { - authority: 2, - round: 1, - digest: BlockDigest::from(digest), - } - }) - .collect(); - - let candidate = CandidateCarrierV1::try_new(carrier, &committee).unwrap(); - assert_eq!(candidate.header().data_acknowledgments().len(), 8_192); - assert!(candidate.canonical_content_bytes().unwrap().len() < MAX_CARRIER_CONTENT_SIZE_V1); - } - - #[test] - fn phase_slot_conflict_is_rejected_even_when_digests_differ() { - let committee = Committee::new_test(vec![1; 4]); - let mut args = args(&committee, 3, 2); - args.phase_batch = vec![ - RbcPhaseStatementV1::Echo { - target: reference(0, 1, 0x01), - }, - RbcPhaseStatementV1::Echo { - target: reference(0, 1, 0x02), - }, - ]; - assert!(matches!( - CandidateCarrierV1::try_new(args, &committee), - Err(RbcDagError::DuplicatePhaseStatement(_)) - )); - } - - #[test] - fn malformed_optional_vertex_does_not_invalidate_outer_candidate() { - let committee = Committee::new_test(vec![1; 4]); - let mut args = args(&committee, 3, 1); - args.consensus_vertex = Some(ConsensusVertexV1::new( - 1, - Vec::new(), - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }, - )); - let candidate = CandidateCarrierV1::try_new(args, &committee).unwrap(); - assert!(matches!( - candidate.validate_consensus_vertex(&committee), - Err(RbcDagProjectionError::InvalidStrongParentThreshold) - )); - } - - #[test] - fn valid_projection_shape_checks_quorum_frontier_and_choice() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - assert!( - candidate - .validate_consensus_vertex(&committee) - .unwrap() - .is_some() - ); - } - - #[test] - fn auth_statement_has_frozen_layout() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - let instance = RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(); - let context = - RbcDagContextV1::new(instance, &committee, BlockAuthenticationScheme::MlDsa65).unwrap(); - let base = context.public_authentication_statement(candidate.reference()); - assert_eq!( - hex::encode(context.committee_id().as_bytes()), - "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62" - ); - assert_eq!( - hex::encode(base), - concat!( - "53544152464953485f5242435f4441475f56310002", - "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", - "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", - "000300000002", - "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f" - ) - ); - assert_eq!( - hex::encode(context.public_authentication_digest(candidate.reference())), - "26a0866c9c6938c9158495f23fd22b281db6371461b6aad109ad41329e5fa5c8" - ); - assert_eq!(&base[..19], AUTHENTICATION_DOMAIN); - assert_eq!(base[19], CARRIER_AUTHENTICATION_KIND); - assert_eq!(base[20], 2); - assert_eq!(&base[21..53], &[0xA5; 32]); - assert_eq!(&base[53..85], context.committee_id().as_bytes()); - assert_eq!(&base[85..87], &3u16.to_be_bytes()); - assert_eq!(&base[87..91], &2u32.to_be_bytes()); - assert_eq!(&base[91..], candidate.reference().digest.as_ref()); - let public_digest = blake3::hash(&base); - for (field, offset) in [ - ("domain", 0), - ("kind", 19), - ("scheme", 20), - ("instance", 21), - ("committee", 53), - ("author", 85), - ("round", 87), - ("content digest", 91), - ] { - let mut changed = base; - changed[offset] ^= 1; - assert_ne!( - blake3::hash(&changed), - public_digest, - "{field} must be bound by the public authenticator" - ); - } - - let mac_context = - RbcDagContextV1::new(instance, &committee, BlockAuthenticationScheme::MacVector) - .unwrap(); - let keyrings = mac_keyrings_for_test(committee.len()); - let mac_statement = mac_context.mac_authentication_statement(candidate.reference(), 2); - assert_eq!( - hex::encode(mac_statement), - concat!( - "53544152464953485f5242435f4441475f56310003", - "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", - "acfb1f9c45727a7366b83e468926bfa9f577cf308078792da0b415d05ae3df62", - "000300000002", - "797b7ffa348c94889c36ea4a0c02070963efe6b7326aaed057f47e825867012f", - "0002" - ) - ); - let key = &keyrings[3][2]; - let tag = key.compute_rbc_tag(&mac_statement); - assert_eq!( - hex::encode(tag.as_ref()), - "118209f3c2c3025918ae7f60fe5a04a94e639cd06d910d89c483035c014fff02" - ); - for (field, offset) in [ - ("domain", 0), - ("kind", 19), - ("scheme", 20), - ("instance", 21), - ("committee", 53), - ("author", 85), - ("round", 87), - ("content digest", 91), - ("recipient", 123), - ] { - let mut changed = mac_statement; - changed[offset] ^= 1; - assert_ne!( - key.compute_rbc_tag(&changed), - tag, - "{field} must be bound by the MAC" - ); - } - } - - #[test] - fn authorizer_is_bound_to_the_claimed_author_and_committee_key() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - let context = RbcDagContextV1::new( - RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::Ed25519, - ) - .unwrap(); - let committee_signer = dummy_signer(); - - assert!(matches!( - context.authenticate( - &candidate, - &committee, - CarrierAuthorizerV1::Ed25519 { - authority: 2, - signer: &committee_signer, - }, - ), - Err(RbcDagError::AuthorizerAuthorityMismatch { - expected: 3, - actual: 2, - }) - )); - - let other_signers = Signer::new_for_test(1); - assert!(matches!( - context.authenticate( - &candidate, - &committee, - CarrierAuthorizerV1::Ed25519 { - authority: 3, - signer: &other_signers[0], - }, - ), - Err(RbcDagError::AuthorizerKeyMismatch) - )); - } - - #[test] - fn every_authentication_scheme_round_trips_and_keeps_content_identity() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - let instance = RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(); - let keyrings = mac_keyrings_for_test(committee.len()); - let ed_signer = dummy_signer(); - let ml44_signer = dummy_ml_dsa_44_signer(); - let ml65_signer = dummy_ml_dsa_65_signer(); - - for (scheme, authorizer) in [ - ( - BlockAuthenticationScheme::Ed25519, - CarrierAuthorizerV1::Ed25519 { - authority: 3, - signer: &ed_signer, - }, - ), - ( - BlockAuthenticationScheme::MlDsa44, - CarrierAuthorizerV1::MlDsa44 { - authority: 3, - signer: &ml44_signer, - }, - ), - ( - BlockAuthenticationScheme::MlDsa65, - CarrierAuthorizerV1::MlDsa65 { - authority: 3, - signer: &ml65_signer, - }, - ), - ( - BlockAuthenticationScheme::MacVector, - CarrierAuthorizerV1::MacVector { - authority: 3, - keys: &keyrings[3], - }, - ), - ] { - let context = RbcDagContextV1::new(instance, &committee, scheme).unwrap(); - let authentication = context - .authenticate(&candidate, &committee, authorizer) - .unwrap(); - let wire = authentication.canonical_wire_bytes(); - let payload_len = match scheme { - BlockAuthenticationScheme::Ed25519 => SIGNATURE_SIZE, - BlockAuthenticationScheme::MlDsa44 => ML_DSA_44_SIGNATURE_SIZE, - BlockAuthenticationScheme::MlDsa65 => ML_DSA_65_SIGNATURE_SIZE, - BlockAuthenticationScheme::MacVector => committee.len() * MAC_TAG_SIZE, - }; - assert_eq!( - &wire[..3], - &[ - CONTENT_FORMAT_FIELD, - CARRIER_FORMAT_VERSION_V1, - authentication_scheme_code(scheme), - ] - ); - assert_eq!(wire.len(), 3 + payload_len); - let decoded = CarrierAuthenticationV1::decode_wire(&wire, &committee).unwrap(); - assert_eq!(decoded, authentication); - let authenticated = context - .verify_authentication(candidate.clone(), decoded, 1, &committee, &keyrings[1]) - .unwrap(); - assert_eq!(authenticated.reference(), candidate.reference()); - assert_eq!(authenticated.receiver(), 1); - assert_eq!(authenticated.context(), context); - } - } - - #[test] - fn mac_verifies_only_local_entry_and_context() { - let committee = Committee::new_test(vec![1; 4]); - let candidate = full_candidate(&committee); - let keyrings = mac_keyrings_for_test(committee.len()); - let instance = RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(); - let context = - RbcDagContextV1::new(instance, &committee, BlockAuthenticationScheme::MacVector) - .unwrap(); - let authentication = context - .authenticate( - &candidate, - &committee, - CarrierAuthorizerV1::MacVector { - authority: 3, - keys: &keyrings[3], - }, - ) - .unwrap(); - let CarrierAuthenticationV1::MacVector(vector) = authentication else { - unreachable!() - }; - let mut poisoned_other = vector.as_bytes().to_vec(); - poisoned_other[2 * MAC_TAG_SIZE] ^= 0xFF; - let poisoned_other = - CarrierAuthenticationV1::MacVector(FlatMacVector::from_bytes(poisoned_other).unwrap()); - assert!( - context - .verify_authentication( - candidate.clone(), - poisoned_other.clone(), - 1, - &committee, - &keyrings[1], - ) - .is_ok() - ); - assert!(matches!( - context.verify_authentication( - candidate.clone(), - poisoned_other, - 2, - &committee, - &keyrings[2], - ), - Err(RbcDagError::InvalidAuthentication) - )); - - let other_context = RbcDagContextV1::new( - RbcDagProtocolInstanceId::new([0xB6; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ) - .unwrap(); - let original = context - .authenticate( - &candidate, - &committee, - CarrierAuthorizerV1::MacVector { - authority: 3, - keys: &keyrings[3], - }, - ) - .unwrap(); - assert!(matches!( - other_context.verify_authentication(candidate, original, 1, &committee, &keyrings[1],), - Err(RbcDagError::InvalidAuthentication) - )); - } - - #[test] - fn candidate_and_capability_are_committee_bound() { - let committee = Committee::new_test(vec![1; 4]); - let other_committee = Committee::new_test(vec![1, 1, 1, 2]); - let candidate = full_candidate(&committee); - let context = RbcDagContextV1::new( - RbcDagProtocolInstanceId::new([0xA5; 32]).unwrap(), - &other_committee, - BlockAuthenticationScheme::Ed25519, - ) - .unwrap(); - assert!(matches!( - context.authenticate( - &candidate, - &other_committee, - CarrierAuthorizerV1::Ed25519 { - authority: 3, - signer: &dummy_signer(), - }, - ), - Err(RbcDagError::CandidateCommitteeMismatch) - )); - assert!(matches!( - candidate.validate_consensus_vertex(&other_committee), - Err(RbcDagProjectionError::CommitteeMismatch) - )); - } - - #[test] - fn sidecar_wire_has_frozen_flat_mac_shape() { - let committee = Committee::new_test(vec![1; 4]); - let tags = (0..4) - .map(|index| MacTag::from_bytes([index; MAC_TAG_SIZE])) - .collect::>(); - let authentication = - CarrierAuthenticationV1::MacVector(FlatMacVector::from_tags(&tags).unwrap()); - let wire = authentication.canonical_wire_bytes(); - assert_eq!(&wire[..3], &[0x00, 0x01, 0x03]); - assert_eq!(wire.len(), 3 + committee.len() * MAC_TAG_SIZE); - assert_eq!( - hex::encode(wire), - concat!( - "000103", - "0000000000000000000000000000000000000000000000000000000000000000", - "0101010101010101010101010101010101010101010101010101010101010101", - "0202020202020202020202020202020202020202020202020202020202020202", - "0303030303030303030303030303030303030303030303030303030303030303" - ) - ); - - let mut trailing = authentication.canonical_wire_bytes(); - trailing.push(0xFF); - assert!(matches!( - CarrierAuthenticationV1::decode_wire(&trailing, &committee), - Err(RbcDagError::TrailingBytes(1)) - )); - let mut truncated = authentication.canonical_wire_bytes(); - truncated.pop(); - assert!(matches!( - CarrierAuthenticationV1::decode_wire(&truncated, &committee), - Err(RbcDagError::UnexpectedEnd) - )); - } -} diff --git a/crates/starfish-core/src/starfish_rbc_dag/model.rs b/crates/starfish-core/src/starfish_rbc_dag/model.rs deleted file mode 100644 index f1506bad..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag/model.rs +++ /dev/null @@ -1,2266 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Deterministic, in-memory model for the embedded-RBC carrier DAG. -//! -//! This reducer intentionally has no networking, storage, timers, or consensus -//! integration. Authentication is represented by a capability supplied by the -//! caller after the outer authenticator has been checked. This keeps the -//! important authority boundary explicit: canonical but unauthenticated -//! content may help header recovery, while only the first authenticated value -//! in a carrier slot is optimistically admitted and allowed to ECHO. - -use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - error::Error, - fmt, - sync::Arc, -}; - -use crate::{ - committee::Committee, - types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, -}; - -use super::{ - AuthenticatedCarrierV1, CandidateCarrierV1, LocallyAuthenticatedCarrierV1, - MAX_PHASE_STATEMENTS_V1, RbcDagCommitteeId, RbcDagContextV1, RbcPhaseStatementV1, - carrier_genesis_reference, -}; - -/// Executable-model runahead bound. This is deliberately a model parameter, -/// not a production protocol constant; the runtime value remains a proof and -/// benchmarking decision. -pub const EXECUTABLE_MODEL_ADMISSION_WINDOW_V1: RoundNumber = 2; -pub const EXECUTABLE_MODEL_BUFFER_WINDOW_V1: RoundNumber = 4; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum IngressAuthentication { - Authenticated, - CandidateOnly, -} - -/// Observable effects of one deterministic reducer transition. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ModelEffect { - /// A threshold is latched, but the exact canonical carrier is absent. - NeedCarrier { - target: BlockReference, - holders: Vec, - }, - /// The local Bracha instance delivered this exact carrier value. - Delivered(BlockReference), - /// One exact author prefix advanced by one carrier. - PrefixAdvanced { - authority: AuthorityIndex, - tip: BlockReference, - }, - /// The sequential fast clock opened the next local carrier round. - CarrierRoundAdvanced(RoundNumber), -} - -/// Snapshot of the lifecycle predicates for one exact carrier. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct CarrierLifecycle { - pub authenticated: bool, - pub admitted: bool, - pub phase_batch_processed: bool, - pub delivered: bool, - pub data_available: bool, - pub prefix_closed: bool, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ModelError { - InvalidCommittee, - CommitteeMismatch { - expected: RbcDagCommitteeId, - actual: RbcDagCommitteeId, - }, - ContextMismatch, - AuthenticationReceiverMismatch { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - UnknownAuthority(AuthorityIndex), - LocalAuthorMismatch { - expected: AuthorityIndex, - actual: AuthorityIndex, - }, - LocalCarrierRequiresStart(BlockReference), - LocalRoundNotOpen(RoundNumber), - UnexpectedLocalRound { - expected: RoundNumber, - actual: RoundNumber, - }, - LocalCarrierAlreadyFixed(RoundNumber), - FutureCarrierOutsideBuffer { - current: RoundNumber, - maximum: RoundNumber, - actual: RoundNumber, - }, - WrongLocalPredecessor { - expected: BlockReference, - actual: BlockReference, - }, - LocalWeakParentNotAdmitted(BlockReference), - LocalPhaseBatchMismatch, - ConflictingCarrierContent(BlockReference), - UnexpectedRecovery(BlockReference), - MissingCarrier(BlockReference), - FrontierLength { - expected: usize, - actual: usize, - }, - FrontierAuthority { - index: AuthorityIndex, - reference: BlockReference, - }, - FrontierNotClosed(BlockReference), - FrontierRegression { - authority: AuthorityIndex, - previous: Option, - proposed: Option, - }, -} - -impl fmt::Display for ModelError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "Starfish-RBC-DAG model error: {self:?}") - } -} - -impl Error for ModelError {} - -#[derive(Clone)] -struct CarrierRecord { - carrier: CandidateCarrierV1, - authenticated: bool, - admitted: bool, - phase_batch_cursor: usize, - delivered: bool, - data_available: bool, - prefix_closed: bool, -} - -impl CarrierRecord { - fn new(carrier: CandidateCarrierV1) -> Self { - Self { - carrier, - authenticated: false, - admitted: false, - phase_batch_cursor: 0, - delivered: false, - data_available: false, - prefix_closed: false, - } - } - - fn lifecycle(&self) -> CarrierLifecycle { - CarrierLifecycle { - authenticated: self.authenticated, - admitted: self.admitted, - phase_batch_processed: self.phase_batch_cursor - == self.carrier.header().phase_batch().len(), - delivered: self.delivered, - data_available: self.data_available, - prefix_closed: self.prefix_closed, - } - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -struct RbcCandidateState { - echoes: BTreeSet, - readies: BTreeSet, - echo_quorum_observed: bool, - ready_validity_observed: bool, - ready_quorum_observed: bool, - requested_holders: BTreeSet, -} - -impl RbcCandidateState { - fn holders(&self) -> BTreeSet { - self.echoes.union(&self.readies).copied().collect() - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -struct RbcSlotState { - echoed: Option, - readied: Option, - delivered: Option, - echo_by_sender: BTreeMap, - ready_by_sender: BTreeMap, - candidates: BTreeMap, -} - -#[derive(Clone, Copy)] -enum RbcAction { - NeedCarrier, - SendReady, - Deliver, - None, -} - -/// Pure local state machine used by the milestone-two simulations. -#[derive(Clone)] -pub struct RbcDagModel { - committee: Arc, - committee_id: RbcDagCommitteeId, - context: RbcDagContextV1, - own_authority: AuthorityIndex, - local_carrier_round: RoundNumber, - own_fixed: BTreeMap, - carriers: BTreeMap, - authenticated_by_slot: BTreeMap<(RoundNumber, AuthorityIndex), BlockReference>, - admitted_by_slot: BTreeMap<(RoundNumber, AuthorityIndex), BlockReference>, - rbc_slots: BTreeMap<(RoundNumber, AuthorityIndex), RbcSlotState>, - pending_phases: VecDeque, - pending_phase_set: BTreeSet, - pending_delivered_batch_replays: VecDeque, - prefix_tips: Vec, - included_frontier: Vec>, - included: BTreeSet, -} - -impl RbcDagModel { - pub fn new( - committee: Arc, - own_authority: AuthorityIndex, - context: RbcDagContextV1, - ) -> Result { - if !committee.known_authority(own_authority) { - return Err(ModelError::UnknownAuthority(own_authority)); - } - let committee_id = - RbcDagCommitteeId::derive(&committee).map_err(|_| ModelError::InvalidCommittee)?; - if context.committee_id() != committee_id { - return Err(ModelError::ContextMismatch); - } - let prefix_tips = committee - .authorities() - .map(carrier_genesis_reference) - .collect(); - Ok(Self { - included_frontier: vec![None; committee.len()], - committee, - committee_id, - context, - own_authority, - local_carrier_round: 1, - own_fixed: BTreeMap::new(), - carriers: BTreeMap::new(), - authenticated_by_slot: BTreeMap::new(), - admitted_by_slot: BTreeMap::new(), - rbc_slots: BTreeMap::new(), - pending_phases: VecDeque::new(), - pending_phase_set: BTreeSet::new(), - pending_delivered_batch_replays: VecDeque::new(), - prefix_tips, - included: BTreeSet::new(), - }) - } - - pub fn own_authority(&self) -> AuthorityIndex { - self.own_authority - } - - pub fn context(&self) -> RbcDagContextV1 { - self.context - } - - /// Current sequential local carrier slot. If `can_create_carrier` is - /// false, the local carrier is fixed and waits for exact-round quorum. - pub fn local_carrier_round(&self) -> RoundNumber { - self.local_carrier_round - } - - pub fn can_create_carrier(&self) -> bool { - !self.own_fixed.contains_key(&self.local_carrier_round) - } - - pub fn pending_phase_batch(&self) -> Vec { - let limit = self - .committee - .len() - .saturating_mul(4) - .min(MAX_PHASE_STATEMENTS_V1); - self.pending_phases - .iter() - .filter(|statement| statement.target().round < self.local_carrier_round) - .take(limit) - .copied() - .collect() - } - - pub fn pending_phase_backlog_len(&self) -> usize { - self.pending_phases.len() - } - - /// Return the exact predecessor and one deterministic quorum of admitted - /// weak parents for the next honest carrier. - pub fn local_parent_set(&self) -> Result<(BlockReference, Vec), ModelError> { - if !self.can_create_carrier() { - return Err(ModelError::LocalCarrierAlreadyFixed( - self.local_carrier_round, - )); - } - let parent_round = self.local_carrier_round - 1; - let own_prev = if parent_round == 0 { - carrier_genesis_reference(self.own_authority) - } else { - *self - .own_fixed - .get(&parent_round) - .ok_or(ModelError::LocalRoundNotOpen(self.local_carrier_round))? - }; - - let mut stake = self.authority_stake(self.own_authority); - let mut weak_parents = Vec::new(); - for authority in self.committee.authorities() { - if authority == self.own_authority { - continue; - } - let parent = if parent_round == 0 { - carrier_genesis_reference(authority) - } else { - let Some(parent) = self - .admitted_by_slot - .get(&(parent_round, authority)) - .copied() - else { - continue; - }; - parent - }; - weak_parents.push(parent); - stake = stake.saturating_add(self.authority_stake(authority)); - if stake >= self.committee.quorum_threshold() { - break; - } - } - if stake < self.committee.quorum_threshold() { - return Err(ModelError::LocalRoundNotOpen(self.local_carrier_round)); - } - Ok((own_prev, weak_parents)) - } - - /// Fix a locally authored carrier. Its phase batch must be the exact - /// bounded FIFO prefix returned by [`Self::pending_phase_batch`]; newly - /// generated ECHO is left for a later carrier. - pub fn start_local_carrier( - &mut self, - authenticated: LocallyAuthenticatedCarrierV1, - ) -> Result, ModelError> { - self.ensure_locally_authenticated(&authenticated)?; - let carrier = authenticated.candidate().clone(); - self.ensure_committee(&carrier)?; - let header = carrier.header(); - if header.author() != self.own_authority { - return Err(ModelError::LocalAuthorMismatch { - expected: self.own_authority, - actual: header.author(), - }); - } - if header.carrier_round() != self.local_carrier_round { - return Err(ModelError::UnexpectedLocalRound { - expected: self.local_carrier_round, - actual: header.carrier_round(), - }); - } - if !self.can_create_carrier() { - return Err(ModelError::LocalCarrierAlreadyFixed( - self.local_carrier_round, - )); - } - let (expected_prev, _) = self.local_parent_set()?; - if header.own_prev() != expected_prev { - return Err(ModelError::WrongLocalPredecessor { - expected: expected_prev, - actual: header.own_prev(), - }); - } - if header.carrier_round() > 1 { - for parent in header.weak_parents() { - if self.admitted_by_slot.get(&(parent.round, parent.authority)) != Some(parent) { - return Err(ModelError::LocalWeakParentNotAdmitted(*parent)); - } - } - } - let expected_phase_batch = self.pending_phase_batch(); - if header.phase_batch() != expected_phase_batch { - return Err(ModelError::LocalPhaseBatchMismatch); - } - - let round = header.carrier_round(); - let selected_phase_indices: BTreeSet<_> = self - .pending_phases - .iter() - .enumerate() - .filter(|(_, statement)| statement.target().round < round) - .take(expected_phase_batch.len()) - .map(|(index, _)| index) - .collect(); - let reference = carrier.reference(); - // Preflight all fallible ingress checks before the proof-critical - // write order. The exact local carrier must be fixed before its local - // ECHO is authorized or any embedded phase statement is exposed. - self.preflight_receive(&carrier)?; - self.own_fixed.insert(round, reference); - for statement in &expected_phase_batch { - self.pending_phase_set.remove(statement); - } - self.pending_phases = self - .pending_phases - .drain(..) - .enumerate() - .filter_map(|(index, statement)| { - (!selected_phase_indices.contains(&index)).then_some(statement) - }) - .collect(); - let mut effects = - self.apply_received_carrier(carrier, IngressAuthentication::Authenticated); - self.maybe_advance_fast_clock(&mut effects); - Ok(effects) - } - - /// Stage canonical content without granting optimistic admission or ECHO. - pub fn stage_candidate( - &mut self, - carrier: CandidateCarrierV1, - ) -> Result, ModelError> { - self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) - } - - /// Admit a carrier only through the opaque capability produced by the - /// codec's context- and receiver-bound authenticator verifier. - pub fn receive_authenticated( - &mut self, - authenticated: AuthenticatedCarrierV1, - ) -> Result, ModelError> { - self.ensure_authenticated(&authenticated)?; - if authenticated.candidate().header().author() == self.own_authority { - return Err(ModelError::LocalCarrierRequiresStart( - authenticated.candidate().reference(), - )); - } - self.receive_carrier( - authenticated.candidate().clone(), - IngressAuthentication::Authenticated, - ) - } - - fn receive_carrier( - &mut self, - carrier: CandidateCarrierV1, - authentication: IngressAuthentication, - ) -> Result, ModelError> { - self.preflight_receive(&carrier)?; - Ok(self.apply_received_carrier(carrier, authentication)) - } - - fn preflight_receive(&self, carrier: &CandidateCarrierV1) -> Result<(), ModelError> { - self.ensure_committee(carrier)?; - let reference = carrier.reference(); - let maximum = self - .local_carrier_round - .saturating_add(EXECUTABLE_MODEL_BUFFER_WINDOW_V1); - if reference.round > maximum { - return Err(ModelError::FutureCarrierOutsideBuffer { - current: self.local_carrier_round, - maximum, - actual: reference.round, - }); - } - let author = carrier.header().author(); - if !self.committee.known_authority(author) { - return Err(ModelError::UnknownAuthority(author)); - } - match self.carriers.get(&reference) { - Some(existing) if &existing.carrier != carrier => { - return Err(ModelError::ConflictingCarrierContent(reference)); - } - Some(_) | None => {} - } - Ok(()) - } - - fn apply_received_carrier( - &mut self, - carrier: CandidateCarrierV1, - authentication: IngressAuthentication, - ) -> Vec { - let reference = carrier.reference(); - self.carriers - .entry(reference) - .or_insert_with(|| CarrierRecord::new(carrier)); - - let mut effects = Vec::new(); - // Canonical content can satisfy a previously latched recovery even if - // the receiver-specific authenticator is invalid. - self.drive_rbc(reference, &mut effects); - - if authentication == IngressAuthentication::Authenticated { - self.carriers - .get_mut(&reference) - .expect("carrier was staged") - .authenticated = true; - let slot_key = (reference.round, reference.authority); - let selected = match self.authenticated_by_slot.get(&slot_key) { - Some(existing) => *existing == reference, - None => { - self.authenticated_by_slot.insert(slot_key, reference); - true - } - }; - if selected && self.in_admission_window(reference.round) { - self.promote_authenticated(reference, &mut effects); - } - } - self.maybe_advance_fast_clock(&mut effects); - self.drain_delivered_phase_batches(&mut effects); - effects - } - - /// Accept an exact recovered carrier only after authenticated phase - /// evidence allocated its candidate. - pub fn recover_carrier( - &mut self, - carrier: CandidateCarrierV1, - ) -> Result, ModelError> { - self.ensure_committee(&carrier)?; - let reference = carrier.reference(); - let key = (reference.round, reference.authority); - let expected = self - .rbc_slots - .get(&key) - .is_some_and(|slot| slot.candidates.contains_key(&reference)); - if !expected { - return Err(ModelError::UnexpectedRecovery(reference)); - } - self.receive_carrier(carrier, IngressAuthentication::CandidateOnly) - } - - /// Record transaction-data availability established by the external - /// Reed-Solomon/reconstruction layer after it verifies this carrier's - /// commitment. Milestone two intentionally treats that layer as a trusted - /// oracle; it does not infer this predicate from unauthenticated ACKs. - pub fn mark_data_available( - &mut self, - reference: BlockReference, - ) -> Result, ModelError> { - self.carriers - .get_mut(&reference) - .ok_or(ModelError::MissingCarrier(reference))? - .data_available = true; - let mut effects = Vec::new(); - self.drive_prefix(reference.authority, &mut effects); - Ok(effects) - } - - pub fn lifecycle(&self, reference: &BlockReference) -> Option { - self.carriers.get(reference).map(CarrierRecord::lifecycle) - } - - pub fn delivered( - &self, - authority: AuthorityIndex, - round: RoundNumber, - ) -> Option { - self.rbc_slots - .get(&(round, authority)) - .and_then(|slot| slot.delivered) - } - - pub fn prefix_tip(&self, authority: AuthorityIndex) -> Option { - let tip = *self.prefix_tips.get(authority as usize)?; - (tip.round > 0).then_some(tip) - } - - pub fn admitted_reference( - &self, - authority: AuthorityIndex, - round: RoundNumber, - ) -> Option { - self.admitted_by_slot.get(&(round, authority)).copied() - } - - /// Include the exact closed-prefix delta named by a committed frontier. - /// The output order is `(round, author, digest)` through `BlockReference`'s - /// canonical ordering. - pub fn apply_frontier( - &mut self, - frontier: &[Option], - ) -> Result, ModelError> { - if frontier.len() != self.committee.len() { - return Err(ModelError::FrontierLength { - expected: self.committee.len(), - actual: frontier.len(), - }); - } - let mut delta = BTreeSet::new(); - for (index, proposed) in frontier.iter().copied().enumerate() { - let authority = index as AuthorityIndex; - if let Some(reference) = proposed { - if reference.authority != authority || reference.round == 0 { - return Err(ModelError::FrontierAuthority { - index: authority, - reference, - }); - } - if !self - .carriers - .get(&reference) - .is_some_and(|record| record.prefix_closed) - { - return Err(ModelError::FrontierNotClosed(reference)); - } - } - let previous = self.included_frontier[index]; - self.collect_frontier_extension(authority, previous, proposed, &mut delta)?; - } - self.included_frontier.clone_from_slice(frontier); - self.included.extend(delta.iter().copied()); - Ok(delta.into_iter().collect()) - } - - fn collect_frontier_extension( - &self, - authority: AuthorityIndex, - previous: Option, - proposed: Option, - delta: &mut BTreeSet, - ) -> Result<(), ModelError> { - let Some(mut cursor) = proposed else { - if previous.is_some() { - return Err(ModelError::FrontierRegression { - authority, - previous, - proposed, - }); - } - return Ok(()); - }; - if Some(cursor) == previous { - return Ok(()); - } - loop { - if Some(cursor) == previous { - return Ok(()); - } - if cursor.round == 0 { - if previous.is_none() && cursor == carrier_genesis_reference(authority) { - return Ok(()); - } - return Err(ModelError::FrontierRegression { - authority, - previous, - proposed, - }); - } - let record = self - .carriers - .get(&cursor) - .ok_or(ModelError::FrontierNotClosed(cursor))?; - if !record.prefix_closed { - return Err(ModelError::FrontierNotClosed(cursor)); - } - delta.insert(cursor); - cursor = record.carrier.header().own_prev(); - } - } - - fn authority_stake(&self, authority: AuthorityIndex) -> Stake { - self.committee.get_stake(authority).unwrap_or(0) - } - - fn ensure_committee(&self, carrier: &CandidateCarrierV1) -> Result<(), ModelError> { - let actual = carrier.committee_id(); - if actual != self.committee_id { - return Err(ModelError::CommitteeMismatch { - expected: self.committee_id, - actual, - }); - } - Ok(()) - } - - fn ensure_authenticated( - &self, - authenticated: &AuthenticatedCarrierV1, - ) -> Result<(), ModelError> { - if authenticated.context() != self.context { - return Err(ModelError::ContextMismatch); - } - if authenticated.receiver() != self.own_authority { - return Err(ModelError::AuthenticationReceiverMismatch { - expected: self.own_authority, - actual: authenticated.receiver(), - }); - } - self.ensure_committee(authenticated.candidate()) - } - - fn ensure_locally_authenticated( - &self, - authenticated: &LocallyAuthenticatedCarrierV1, - ) -> Result<(), ModelError> { - if authenticated.context() != self.context { - return Err(ModelError::ContextMismatch); - } - self.ensure_committee(authenticated.candidate()) - } - - fn voters_stake(&self, voters: &BTreeSet) -> Stake { - voters.iter().fold(0, |stake, authority| { - stake.saturating_add(self.authority_stake(*authority)) - }) - } - - fn rbc_slot_mut(&mut self, reference: BlockReference) -> &mut RbcSlotState { - self.rbc_slots - .entry((reference.round, reference.authority)) - .or_default() - } - - fn authorize_local_echo(&mut self, reference: BlockReference, effects: &mut Vec) { - let own = self.own_authority; - let slot = self.rbc_slot_mut(reference); - if slot.echoed.is_some() { - return; - } - slot.echoed = Some(reference); - slot.echo_by_sender.insert(own, reference); - slot.candidates - .entry(reference) - .or_default() - .echoes - .insert(own); - self.queue_local_phase(RbcPhaseStatementV1::Echo { target: reference }); - self.drive_rbc(reference, effects); - } - - fn queue_local_phase(&mut self, statement: RbcPhaseStatementV1) { - if self.pending_phase_set.insert(statement) { - self.pending_phases.push_back(statement); - } - } - - fn process_phase_batch(&mut self, outer: BlockReference, effects: &mut Vec) { - self.process_phase_batch_steps(outer, usize::MAX, effects); - self.drain_delivered_phase_batches(effects); - } - - fn drain_delivered_phase_batches(&mut self, effects: &mut Vec) { - while let Some(outer) = self.pending_delivered_batch_replays.pop_front() { - self.process_phase_batch_steps(outer, usize::MAX, effects); - } - } - - fn process_phase_batch_steps( - &mut self, - outer: BlockReference, - maximum_steps: usize, - effects: &mut Vec, - ) { - let mut processed = 0; - loop { - if processed == maximum_steps { - return; - } - let Some((sender, statement)) = self.carriers.get(&outer).and_then(|record| { - record - .carrier - .header() - .phase_batch() - .get(record.phase_batch_cursor) - .copied() - .map(|statement| (record.carrier.header().author(), statement)) - }) else { - return; - }; - // Applying the statement is idempotent. Advance the persisted - // cursor only afterwards, so a crash between the two replays the - // same statement rather than skipping the unprocessed tail. - self.record_phase(sender, statement, effects); - self.carriers - .get_mut(&outer) - .expect("the outer carrier remains pinned") - .phase_batch_cursor += 1; - processed += 1; - } - } - - fn record_phase( - &mut self, - sender: AuthorityIndex, - statement: RbcPhaseStatementV1, - effects: &mut Vec, - ) { - if !self.committee.known_authority(sender) { - return; - } - let target = statement.target(); - if sender == self.own_authority { - let authorized = self - .rbc_slots - .get(&(target.round, target.authority)) - .is_some_and(|slot| match statement { - RbcPhaseStatementV1::Echo { .. } => slot.echoed == Some(target), - RbcPhaseStatementV1::Ready { .. } => slot.readied == Some(target), - }); - if !authorized { - // An own-authored embedded statement is replay, not fresh - // authority. The corresponding persisted local lock must - // already exist before it may reconstruct sender evidence. - return; - } - } - let slot = self.rbc_slot_mut(target); - let senders = match statement { - RbcPhaseStatementV1::Echo { .. } => &mut slot.echo_by_sender, - RbcPhaseStatementV1::Ready { .. } => &mut slot.ready_by_sender, - }; - match senders.get(&sender) { - Some(existing) if *existing != target => return, - Some(_) => return, - None => { - senders.insert(sender, target); - } - } - let candidate = slot.candidates.entry(target).or_default(); - match statement { - RbcPhaseStatementV1::Echo { .. } => { - candidate.echoes.insert(sender); - } - RbcPhaseStatementV1::Ready { .. } => { - candidate.readies.insert(sender); - } - } - self.drive_rbc(target, effects); - } - - fn drive_rbc(&mut self, target: BlockReference, effects: &mut Vec) { - let slot_key = (target.round, target.authority); - if !self - .rbc_slots - .get(&slot_key) - .is_some_and(|slot| slot.candidates.contains_key(&target)) - { - // Merely staging canonical content is not authenticated RBC - // evidence. Candidate state is allocated only by a locally - // authorized ECHO or an embedded ECHO/READY statement. - return; - } - loop { - let header_available = self.carriers.contains_key(&target); - let q = self.committee.quorum_threshold(); - let v = self.committee.validity_threshold(); - let action = { - let echo_stake; - let ready_stake; - { - let slot = self.rbc_slot_mut(target); - let candidate = slot.candidates.entry(target).or_default(); - echo_stake = candidate.echoes.clone(); - ready_stake = candidate.readies.clone(); - } - let echo_stake = self.voters_stake(&echo_stake); - let ready_stake = self.voters_stake(&ready_stake); - let slot = self.rbc_slot_mut(target); - let candidate = slot.candidates.entry(target).or_default(); - candidate.echo_quorum_observed |= echo_stake >= q; - candidate.ready_validity_observed |= ready_stake >= v; - candidate.ready_quorum_observed |= ready_stake >= q; - let ready_trigger = - candidate.echo_quorum_observed || candidate.ready_validity_observed; - let needs_header = !header_available - && ((slot.readied.is_none() && ready_trigger) - || (slot.delivered.is_none() && candidate.ready_quorum_observed)); - if needs_header { - let holders = candidate.holders(); - if holders != candidate.requested_holders { - candidate.requested_holders = holders; - RbcAction::NeedCarrier - } else { - RbcAction::None - } - } else if header_available && slot.readied.is_none() && ready_trigger { - RbcAction::SendReady - } else if header_available - && slot.delivered.is_none() - && candidate.ready_quorum_observed - { - RbcAction::Deliver - } else { - RbcAction::None - } - }; - - match action { - RbcAction::NeedCarrier => { - let holders = self - .rbc_slots - .get(&(target.round, target.authority)) - .and_then(|slot| slot.candidates.get(&target)) - .map(RbcCandidateState::holders) - .unwrap_or_default() - .into_iter() - .collect(); - effects.push(ModelEffect::NeedCarrier { target, holders }); - break; - } - RbcAction::SendReady => { - let own = self.own_authority; - let slot = self.rbc_slot_mut(target); - slot.readied = Some(target); - slot.ready_by_sender.insert(own, target); - slot.candidates - .entry(target) - .or_default() - .readies - .insert(own); - self.queue_local_phase(RbcPhaseStatementV1::Ready { target }); - } - RbcAction::Deliver => { - self.rbc_slot_mut(target).delivered = Some(target); - let record = self - .carriers - .get_mut(&target) - .expect("delivery requires exact canonical carrier content"); - record.delivered = true; - effects.push(ModelEffect::Delivered(target)); - self.pending_delivered_batch_replays.push_back(target); - self.drive_prefix(target.authority, effects); - } - RbcAction::None => break, - } - } - } - - fn maybe_advance_fast_clock(&mut self, effects: &mut Vec) { - let round = self.local_carrier_round; - if !self.own_fixed.contains_key(&round) { - return; - } - let admitted: BTreeSet<_> = self - .admitted_by_slot - .keys() - .filter_map(|(candidate_round, authority)| { - (*candidate_round == round).then_some(*authority) - }) - .collect(); - if self.voters_stake(&admitted) < self.committee.quorum_threshold() { - return; - } - self.local_carrier_round = round.saturating_add(1); - effects.push(ModelEffect::CarrierRoundAdvanced(self.local_carrier_round)); - self.promote_buffered_window(effects); - } - - fn in_admission_window(&self, round: RoundNumber) -> bool { - round - <= self - .local_carrier_round - .saturating_add(EXECUTABLE_MODEL_ADMISSION_WINDOW_V1) - } - - fn promote_authenticated(&mut self, reference: BlockReference, effects: &mut Vec) { - let slot_key = (reference.round, reference.authority); - if self.authenticated_by_slot.get(&slot_key) != Some(&reference) - || self - .carriers - .get(&reference) - .is_none_or(|record| record.admitted) - { - return; - } - self.admitted_by_slot.insert(slot_key, reference); - self.carriers - .get_mut(&reference) - .expect("an authenticated carrier remains staged") - .admitted = true; - self.authorize_local_echo(reference, effects); - self.process_phase_batch(reference, effects); - } - - fn promote_buffered_window(&mut self, effects: &mut Vec) { - let eligible: Vec<_> = self - .authenticated_by_slot - .values() - .copied() - .filter(|reference| self.in_admission_window(reference.round)) - .collect(); - for reference in eligible { - self.promote_authenticated(reference, effects); - } - } - - fn drive_prefix(&mut self, authority: AuthorityIndex, effects: &mut Vec) { - loop { - let Some(current_tip) = self.prefix_tips.get(authority as usize).copied() else { - return; - }; - let Some(next_round) = current_tip.round.checked_add(1) else { - return; - }; - let Some(next) = self.delivered(authority, next_round) else { - return; - }; - let can_close = self.carriers.get(&next).is_some_and(|record| { - record.delivered - && record.data_available - && record.carrier.header().own_prev() == current_tip - }); - if !can_close { - return; - } - self.carriers - .get_mut(&next) - .expect("delivered carrier exists") - .prefix_closed = true; - self.prefix_tips[authority as usize] = next; - effects.push(ModelEffect::PrefixAdvanced { - authority, - tip: next, - }); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - crypto::{TransactionsCommitment, mac_keyrings_for_test}, - starfish_rbc_dag::{CarrierHeaderV1Args, RbcDagError}, - types::{BlockAuthenticationScheme, BlockReference}, - }; - - fn committee(n: usize) -> Arc { - Committee::new_test(vec![1; n]) - } - - fn context(committee: &Committee) -> RbcDagContextV1 { - RbcDagContextV1::new( - super::super::RbcDagProtocolInstanceId::new([0xD1; 32]).unwrap(), - committee, - BlockAuthenticationScheme::MacVector, - ) - .unwrap() - } - - fn model(committee: Arc, authority: AuthorityIndex) -> RbcDagModel { - let context = context(&committee); - RbcDagModel::new(committee, authority, context).unwrap() - } - - fn authenticate_for( - committee: &Committee, - carrier: &CandidateCarrierV1, - receiver: AuthorityIndex, - ) -> AuthenticatedCarrierV1 { - let context = context(committee); - let keyrings = mac_keyrings_for_test(committee.len()); - let author = carrier.header().author() as usize; - let authentication = context - .authenticate( - carrier, - committee, - super::super::CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ) - .unwrap(); - context - .verify_authentication( - carrier.clone(), - authentication, - receiver, - committee, - &keyrings[receiver as usize], - ) - .unwrap() - } - - fn authenticate_local( - committee: &Committee, - carrier: &CandidateCarrierV1, - ) -> LocallyAuthenticatedCarrierV1 { - let context = context(committee); - let keyrings = mac_keyrings_for_test(committee.len()); - let author = carrier.header().author() as usize; - context - .authenticate_local( - carrier.clone(), - committee, - super::super::CarrierAuthorizerV1::MacVector { - authority: author as AuthorityIndex, - keys: &keyrings[author], - }, - ) - .unwrap() - } - - fn admit( - model: &mut RbcDagModel, - carrier: CandidateCarrierV1, - ) -> Result, ModelError> { - let authenticated = authenticate_for(&model.committee, &carrier, model.own_authority); - model.receive_authenticated(authenticated) - } - - fn candidate( - committee: &Committee, - author: AuthorityIndex, - round: RoundNumber, - own_prev: BlockReference, - weak_parents: Vec, - phase_batch: Vec, - marker: u64, - ) -> Result { - CandidateCarrierV1::try_new( - CarrierHeaderV1Args { - author, - carrier_round: round, - own_prev, - weak_parents, - transactions_commitment: TransactionsCommitment::default(), - data_acknowledgments: Vec::new(), - phase_batch, - consensus_vertex: None, - creation_time_ns: marker, - }, - committee, - ) - } - - fn genesis_parents( - committee: &Committee, - author: AuthorityIndex, - ) -> (BlockReference, Vec) { - let own = carrier_genesis_reference(author); - let mut stake = committee.get_stake(author).unwrap(); - let mut weak = Vec::new(); - for other in committee.authorities() { - if other == author { - continue; - } - weak.push(carrier_genesis_reference(other)); - stake += committee.get_stake(other).unwrap(); - if stake >= committee.quorum_threshold() { - break; - } - } - (own, weak) - } - - fn build_local(model: &mut RbcDagModel, marker: u64) -> CandidateCarrierV1 { - let (own_prev, weak_parents) = model.local_parent_set().unwrap(); - let carrier = candidate( - &model.committee, - model.own_authority, - model.local_carrier_round, - own_prev, - weak_parents, - model.pending_phase_batch(), - marker, - ) - .unwrap(); - let authenticated = authenticate_local(&model.committee, &carrier); - model.start_local_carrier(authenticated).unwrap(); - carrier - } - - fn run_honest_round(models: &mut [RbcDagModel], round: RoundNumber) -> Vec { - let carriers: Vec<_> = models - .iter_mut() - .enumerate() - .map(|(index, model)| build_local(model, u64::from(round) * 100 + index as u64)) - .collect(); - for carrier in &carriers { - for model in models.iter_mut() { - if model.own_authority != carrier.header().author() { - let authenticated = - authenticate_for(&model.committee, carrier, model.own_authority); - model.receive_authenticated(authenticated).unwrap(); - } - model.mark_data_available(carrier.reference()).unwrap(); - } - } - assert!( - models - .iter() - .all(|model| model.local_carrier_round() == round + 1) - ); - carriers - } - - fn all_honest_progress(n: usize) { - let committee = committee(n); - let mut models: Vec<_> = committee - .authorities() - .map(|authority| model(Arc::clone(&committee), authority)) - .collect(); - let mut rounds = Vec::new(); - for round in 1..=6 { - rounds.push(run_honest_round(&mut models, round)); - } - for model in &models { - for carriers in rounds.iter().take(4) { - for carrier in carriers { - assert_eq!( - model - .delivered(carrier.header().author(), carrier.header().carrier_round()), - Some(carrier.reference()) - ); - assert!(model.lifecycle(&carrier.reference()).unwrap().prefix_closed); - } - } - } - } - - #[test] - fn four_node_heartbeat_only_run_delivers_every_mature_carrier() { - all_honest_progress(4); - } - - #[test] - fn seven_node_heartbeat_only_run_delivers_every_mature_carrier() { - all_honest_progress(7); - } - - #[test] - fn phase_backlog_exposes_only_a_bounded_fifo_prefix() { - let committee = committee(4); - let mut model = model(committee, 0); - model.local_carrier_round = 4; - let mut queued = Vec::new(); - for round in 1..=3 { - for author in 0..4 { - let target = BlockReference::new_test(author, round); - queued.push(RbcPhaseStatementV1::Echo { target }); - queued.push(RbcPhaseStatementV1::Ready { target }); - } - } - for statement in &queued { - model.queue_local_phase(*statement); - } - - assert_eq!(model.pending_phase_backlog_len(), 24); - assert_eq!(model.pending_phase_batch(), queued[..16]); - } - - #[test] - fn local_carrier_must_drain_the_exact_bounded_phase_prefix() { - let committee = committee(4); - let mut models: Vec<_> = (0..4) - .map(|authority| model(Arc::clone(&committee), authority)) - .collect(); - run_honest_round(&mut models, 1); - - let model = &mut models[0]; - let expected = model.pending_phase_batch(); - assert!(!expected.is_empty()); - let (own_prev, weak_parents) = model.local_parent_set().unwrap(); - let omitted = candidate( - &committee, - model.own_authority, - model.local_carrier_round, - own_prev, - weak_parents.clone(), - Vec::new(), - 0xA1, - ) - .unwrap(); - let omitted_authentication = authenticate_local(&committee, &omitted); - assert_eq!( - model.start_local_carrier(omitted_authentication), - Err(ModelError::LocalPhaseBatchMismatch) - ); - assert!(model.can_create_carrier()); - - let exact = candidate( - &committee, - model.own_authority, - model.local_carrier_round, - own_prev, - weak_parents, - expected, - 0xA2, - ) - .unwrap(); - let exact_authentication = authenticate_local(&committee, &exact); - model.start_local_carrier(exact_authentication).unwrap(); - assert_eq!(model.own_fixed.get(&2), Some(&exact.reference())); - } - - fn record_phase( - model: &mut RbcDagModel, - sender: AuthorityIndex, - statement: RbcPhaseStatementV1, - ) -> Vec { - let mut effects = Vec::new(); - model.record_phase(sender, statement, &mut effects); - model.drain_delivered_phase_batches(&mut effects); - effects - } - - fn force_deliver(model: &mut RbcDagModel, carrier: CandidateCarrierV1) { - let target = carrier.reference(); - model.stage_candidate(carrier).unwrap(); - let senders: Vec<_> = model - .committee - .authorities() - .filter(|sender| *sender != model.own_authority) - .take(model.committee.quorum_threshold() as usize) - .collect(); - for sender in senders { - record_phase(model, sender, RbcPhaseStatementV1::Ready { target }); - } - assert_eq!( - model.delivered(target.authority, target.round), - Some(target) - ); - } - - #[test] - fn threshold_before_header_requests_then_recovers_exact_carrier() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 1).unwrap(); - let target = carrier.reference(); - - assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target }).is_empty()); - assert!(record_phase(&mut model, 1, RbcPhaseStatementV1::Echo { target }).is_empty()); - assert!(matches!( - record_phase( - &mut model, - 2, - RbcPhaseStatementV1::Echo { target } - ) - .as_slice(), - [ModelEffect::NeedCarrier { target: requested, holders }] - if *requested == target && holders == &[0, 1, 2] - )); - - model.recover_carrier(carrier).unwrap(); - assert!( - model - .pending_phases - .contains(&RbcPhaseStatementV1::Ready { target }) - ); - let lifecycle = model.lifecycle(&target).unwrap(); - assert!(!lifecycle.authenticated); - assert!(!lifecycle.admitted); - } - - #[test] - fn staged_content_without_phase_evidence_cannot_authorize_recovery() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 2).unwrap(); - let reference = carrier.reference(); - - model.stage_candidate(carrier.clone()).unwrap(); - assert!(model.lifecycle(&reference).is_some()); - assert!(model.rbc_slots.is_empty()); - assert_eq!( - model.recover_carrier(carrier), - Err(ModelError::UnexpectedRecovery(reference)) - ); - assert!(model.rbc_slots.is_empty()); - } - - #[test] - fn ready_threshold_before_content_recovers_then_delivers() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 3).unwrap(); - let target = carrier.reference(); - - assert!(record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target }).is_empty()); - assert!(matches!( - record_phase( - &mut model, - 1, - RbcPhaseStatementV1::Ready { target } - ) - .as_slice(), - [ModelEffect::NeedCarrier { target: requested, holders }] - if *requested == target && holders == &[0, 1] - )); - assert_eq!(model.delivered(0, 1), None); - assert!(model.lifecycle(&target).is_none()); - - let effects = model.recover_carrier(carrier).unwrap(); - assert!(effects.contains(&ModelEffect::Delivered(target))); - assert_eq!(model.delivered(0, 1), Some(target)); - assert!( - model - .pending_phases - .contains(&RbcPhaseStatementV1::Ready { target }) - ); - } - - #[test] - fn cross_committee_candidate_is_rejected_before_state_mutation() { - let local_committee = committee(4); - let foreign_committee = Committee::new_test(vec![2; 4]); - let mut model = model(Arc::clone(&local_committee), 3); - let (own_prev, weak) = genesis_parents(&foreign_committee, 0); - let foreign = candidate(&foreign_committee, 0, 1, own_prev, weak, Vec::new(), 9).unwrap(); - let reference = foreign.reference(); - - assert!(matches!( - model.stage_candidate(foreign), - Err(ModelError::CommitteeMismatch { .. }) - )); - assert!(model.lifecycle(&reference).is_none()); - assert!(model.admitted_reference(0, 1).is_none()); - assert!(model.rbc_slots.is_empty()); - } - - #[test] - fn authenticated_capability_is_bound_to_context_and_receiver() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 10).unwrap(); - let reference = carrier.reference(); - - let wrong_receiver = authenticate_for(&committee, &carrier, 2); - assert_eq!( - model.receive_authenticated(wrong_receiver), - Err(ModelError::AuthenticationReceiverMismatch { - expected: 3, - actual: 2, - }) - ); - - let other_context = RbcDagContextV1::new( - super::super::RbcDagProtocolInstanceId::new([0xD2; 32]).unwrap(), - &committee, - BlockAuthenticationScheme::MacVector, - ) - .unwrap(); - let keyrings = mac_keyrings_for_test(committee.len()); - let authentication = other_context - .authenticate( - &carrier, - &committee, - super::super::CarrierAuthorizerV1::MacVector { - authority: 0, - keys: &keyrings[0], - }, - ) - .unwrap(); - let wrong_context = other_context - .verify_authentication(carrier, authentication, 3, &committee, &keyrings[3]) - .unwrap(); - assert_eq!( - model.receive_authenticated(wrong_context), - Err(ModelError::ContextMismatch) - ); - assert!(model.lifecycle(&reference).is_none()); - assert!(model.rbc_slots.is_empty()); - } - - #[test] - fn locally_authored_carrier_can_only_enter_through_atomic_start() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 0); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 11).unwrap(); - let reference = carrier.reference(); - let authenticated = authenticate_for(&committee, &carrier, 0); - - assert_eq!( - model.receive_authenticated(authenticated), - Err(ModelError::LocalCarrierRequiresStart(reference)) - ); - assert!(model.lifecycle(&reference).is_none()); - assert!(model.own_fixed.is_empty()); - assert!(model.rbc_slots.is_empty()); - } - - #[test] - fn own_embedded_phase_requires_the_persisted_local_lock() { - let committee = committee(4); - let mut model = model(committee, 3); - let target = BlockReference::new_test(0, 1); - - assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Echo { target }).is_empty()); - assert!(record_phase(&mut model, 3, RbcPhaseStatementV1::Ready { target }).is_empty()); - assert!(model.rbc_slots.is_empty()); - } - - #[test] - fn phase_replay_and_equivocation_count_each_sender_once() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let first = BlockReference::new_test(0, 1); - let conflicting = BlockReference::new_test(0, 1); - let mut conflicting = conflicting; - conflicting.digest = crate::types::BlockDigest::from([0x77; 32]); - - for _ in 0..3 { - assert!( - record_phase(&mut model, 0, RbcPhaseStatementV1::Echo { target: first }).is_empty() - ); - } - assert!( - record_phase( - &mut model, - 0, - RbcPhaseStatementV1::Echo { - target: conflicting - } - ) - .is_empty() - ); - assert!( - record_phase(&mut model, 1, RbcPhaseStatementV1::Echo { target: first }).is_empty() - ); - assert!(matches!( - record_phase( - &mut model, - 2, - RbcPhaseStatementV1::Echo { target: first } - ) - .as_slice(), - [ModelEffect::NeedCarrier { target, .. }] if *target == first - )); - let slot = model.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.echo_by_sender.len(), 3); - assert_eq!(slot.echo_by_sender[&0], first); - assert!(!slot.candidates.contains_key(&conflicting)); - - record_phase(&mut model, 0, RbcPhaseStatementV1::Ready { target: first }); - record_phase( - &mut model, - 0, - RbcPhaseStatementV1::Ready { - target: conflicting, - }, - ); - let slot = model.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.ready_by_sender[&0], first); - assert!(!slot.candidates.contains_key(&conflicting)); - } - - #[test] - fn split_initial_values_converge_on_one_delivery() { - let committee = committee(4); - let (own_prev, weak) = genesis_parents(&committee, 0); - let first = candidate(&committee, 0, 1, own_prev, weak.clone(), Vec::new(), 11).unwrap(); - let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 12).unwrap(); - let first_ref = first.reference(); - let conflicting_ref = conflicting.reference(); - let mut models: Vec<_> = committee - .authorities() - .map(|authority| model(Arc::clone(&committee), authority)) - .collect(); - - for (index, model) in models.iter_mut().enumerate() { - let (admitted, staged) = if index == 3 { - (conflicting.clone(), first.clone()) - } else { - (first.clone(), conflicting.clone()) - }; - if model.own_authority == admitted.header().author() { - // The dealer is Byzantine in this trace. Its local behavior is - // outside the honest local-start API, so retain both bytes and - // drive its receive-side RBC state only from phase evidence. - model.stage_candidate(admitted).unwrap(); - model.stage_candidate(staged).unwrap(); - } else { - admit(model, admitted).unwrap(); - model.stage_candidate(staged).unwrap(); - } - } - - // Authorities 0, 1, and 2 ECHO the first value; authority 3 ECHOs the - // conflicting value. The first value reaches Q and READY amplification - // carries the receiver that saw the split INIT to the same delivery. - for model in &mut models { - for sender in 0..3 { - record_phase( - model, - sender, - RbcPhaseStatementV1::Echo { target: first_ref }, - ); - } - record_phase( - model, - 3, - RbcPhaseStatementV1::Echo { - target: conflicting_ref, - }, - ); - } - for model in &mut models { - for sender in 0..3 { - record_phase( - model, - sender, - RbcPhaseStatementV1::Ready { target: first_ref }, - ); - } - } - assert!( - models - .iter() - .all(|model| model.delivered(0, 1) == Some(first_ref)) - ); - assert!( - models - .iter() - .all(|model| model.delivered(0, 1) != Some(conflicting_ref)) - ); - } - - #[test] - fn typed_outer_carriers_enforce_split_value_phase_locks_end_to_end() { - let committee = committee(4); - let (own_prev, weak) = genesis_parents(&committee, 0); - let first = candidate(&committee, 0, 1, own_prev, weak.clone(), Vec::new(), 21).unwrap(); - let conflicting = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 22).unwrap(); - let first_ref = first.reference(); - let conflicting_ref = conflicting.reference(); - let mut receiver = model(Arc::clone(&committee), 3); - - admit(&mut receiver, first).unwrap(); - admit(&mut receiver, conflicting).unwrap(); - assert_eq!(receiver.admitted_reference(0, 1), Some(first_ref)); - - let outer = |author: AuthorityIndex, - round: RoundNumber, - phase_batch: Vec, - marker: u64| { - let own_prev = BlockReference::new_test(author, round - 1); - let weak_parents = (0..4) - .filter(|other| *other != author) - .take(2) - .map(|other| BlockReference::new_test(other, round - 1)) - .collect(); - candidate( - &committee, - author, - round, - own_prev, - weak_parents, - phase_batch, - marker, - ) - .unwrap() - }; - - for author in 0..3 { - let carrier = outer( - author, - 2, - vec![RbcPhaseStatementV1::Echo { target: first_ref }], - 200 + u64::from(author), - ); - admit(&mut receiver, carrier).unwrap(); - } - for author in 0..3 { - let mut phase_batch = Vec::new(); - if author == 0 { - phase_batch.push(RbcPhaseStatementV1::Echo { - target: conflicting_ref, - }); - } - phase_batch.push(RbcPhaseStatementV1::Ready { target: first_ref }); - let carrier = outer(author, 3, phase_batch, 300 + u64::from(author)); - admit(&mut receiver, carrier).unwrap(); - } - - assert_eq!(receiver.delivered(0, 1), Some(first_ref)); - assert_ne!(receiver.delivered(0, 1), Some(conflicting_ref)); - let slot = receiver.rbc_slots.get(&(1, 0)).unwrap(); - assert_eq!(slot.echo_by_sender.get(&0), Some(&first_ref)); - assert!(!slot.candidates.contains_key(&conflicting_ref)); - } - - #[test] - fn non_equivocating_phase_reordering_has_the_same_result() { - let committee = committee(4); - let (own_prev, weak) = genesis_parents(&committee, 0); - let carrier = candidate(&committee, 0, 1, own_prev, weak, Vec::new(), 15).unwrap(); - let target = carrier.reference(); - let mut left = model(Arc::clone(&committee), 3); - let mut right = model(Arc::clone(&committee), 3); - left.stage_candidate(carrier.clone()).unwrap(); - right.stage_candidate(carrier).unwrap(); - - for sender in [0, 1, 2] { - record_phase(&mut left, sender, RbcPhaseStatementV1::Echo { target }); - } - for sender in [2, 0, 1] { - record_phase(&mut right, sender, RbcPhaseStatementV1::Echo { target }); - } - for sender in [0, 1, 2] { - record_phase(&mut left, sender, RbcPhaseStatementV1::Ready { target }); - } - for sender in [1, 2, 0] { - record_phase(&mut right, sender, RbcPhaseStatementV1::Ready { target }); - } - assert_eq!(left.delivered(0, 1), Some(target)); - assert_eq!(right.delivered(0, 1), Some(target)); - assert_eq!(left.pending_phase_batch(), right.pending_phase_batch()); - } - - #[test] - fn delivered_candidate_replays_batch_even_without_admission() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let target = BlockReference::new_test(1, 1); - let own_prev = BlockReference::new_test(0, 1); - let weak = vec![ - BlockReference::new_test(1, 1), - BlockReference::new_test(2, 1), - ]; - let outer = candidate( - &committee, - 0, - 2, - own_prev, - weak, - vec![RbcPhaseStatementV1::Echo { target }], - 2, - ) - .unwrap(); - let outer_ref = outer.reference(); - model.stage_candidate(outer.clone()).unwrap(); - assert!( - !model - .rbc_slots - .get(&(target.round, target.authority)) - .is_some_and(|slot| slot.echo_by_sender.contains_key(&0)) - ); - - force_deliver(&mut model, outer); - assert_eq!( - model.rbc_slots[&(target.round, target.authority)].echo_by_sender[&0], - target - ); - let lifecycle = model.lifecycle(&outer_ref).unwrap(); - assert!(!lifecycle.admitted); - assert!(lifecycle.delivered); - assert!(lifecycle.phase_batch_processed); - } - - #[test] - fn replay_after_crash_before_batch_cursor_does_not_skip_the_tail() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let own_prev = BlockReference::new_test(0, 1); - let weak = vec![ - BlockReference::new_test(1, 1), - BlockReference::new_test(2, 1), - ]; - let first = BlockReference::new_test(1, 1); - let second = BlockReference::new_test(2, 1); - let outer = candidate( - &committee, - 0, - 2, - own_prev, - weak, - vec![ - RbcPhaseStatementV1::Echo { target: first }, - RbcPhaseStatementV1::Ready { target: second }, - ], - 0xCA, - ) - .unwrap(); - let outer_ref = outer.reference(); - model.stage_candidate(outer).unwrap(); - - let mut uninterrupted = model.clone(); - uninterrupted.process_phase_batch(outer_ref, &mut Vec::new()); - - // Model a crash after the first idempotent statement was persisted but - // before the outer batch cursor was advanced. - let mut restarted = model; - restarted.record_phase( - 0, - RbcPhaseStatementV1::Echo { target: first }, - &mut Vec::new(), - ); - restarted.process_phase_batch(outer_ref, &mut Vec::new()); - - assert_eq!(restarted.rbc_slots, uninterrupted.rbc_slots); - assert_eq!(restarted.pending_phases, uninterrupted.pending_phases); - assert!( - restarted - .lifecycle(&outer_ref) - .unwrap() - .phase_batch_processed - ); - } - - #[test] - fn deeply_chained_delivered_batches_use_an_explicit_work_queue() { - const DEPTH: RoundNumber = 2_048; - - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let mut previous = None; - let mut references = Vec::with_capacity(DEPTH as usize); - for round in 1..=DEPTH { - let own_prev = previous.unwrap_or_else(|| carrier_genesis_reference(0)); - let parent = |authority| { - if round == 1 { - carrier_genesis_reference(authority) - } else { - BlockReference::new_test(authority, round - 1) - } - }; - let phase_batch = previous - .map(|target| vec![RbcPhaseStatementV1::Ready { target }]) - .unwrap_or_default(); - let carrier = candidate( - &committee, - 0, - round, - own_prev, - vec![parent(1), parent(2)], - phase_batch, - u64::from(round), - ) - .unwrap(); - let reference = carrier.reference(); - model - .carriers - .insert(reference, CarrierRecord::new(carrier)); - let mut candidate_state = RbcCandidateState::default(); - candidate_state.readies.extend([1, 2]); - let mut slot = RbcSlotState::default(); - slot.ready_by_sender - .extend([(1, reference), (2, reference)]); - slot.candidates.insert(reference, candidate_state); - model.rbc_slots.insert((round, 0), slot); - references.push(reference); - previous = Some(reference); - } - - model - .pending_delivered_batch_replays - .push_back(*references.last().unwrap()); - let mut effects = Vec::new(); - model.drain_delivered_phase_batches(&mut effects); - - assert!(model.pending_delivered_batch_replays.is_empty()); - assert_eq!(model.delivered(0, 1), Some(references[0])); - assert_eq!( - model.delivered(0, DEPTH - 1), - Some(references[DEPTH as usize - 2]) - ); - assert!(model.delivered(0, DEPTH).is_none()); - } - - #[test] - fn quorum_of_future_carriers_cannot_jump_the_sequential_clock() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 0); - let mut future_references = Vec::new(); - for author in 1..4 { - let own_prev = BlockReference::new_test(author, 4); - let weak = (0..4) - .filter(|other| *other != author) - .take(2) - .map(|other| BlockReference::new_test(other, 4)) - .collect(); - let future = candidate( - &committee, - author, - 5, - own_prev, - weak, - Vec::new(), - u64::from(author), - ) - .unwrap(); - future_references.push(future.reference()); - admit(&mut model, future).unwrap(); - } - assert_eq!(model.local_carrier_round(), 1); - assert!(model.can_create_carrier()); - for reference in future_references { - let lifecycle = model.lifecycle(&reference).unwrap(); - assert!(lifecycle.authenticated); - assert!(!lifecycle.admitted); - assert!( - !model - .rbc_slots - .contains_key(&(reference.round, reference.authority)) - ); - } - } - - #[test] - fn carrier_beyond_the_bounded_future_buffer_is_rejected_without_state() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 0); - let carrier = candidate( - &committee, - 1, - 6, - BlockReference::new_test(1, 5), - vec![ - BlockReference::new_test(0, 5), - BlockReference::new_test(2, 5), - ], - Vec::new(), - 60, - ) - .unwrap(); - let reference = carrier.reference(); - let authenticated = authenticate_for(&committee, &carrier, 0); - - assert_eq!( - model.receive_authenticated(authenticated), - Err(ModelError::FutureCarrierOutsideBuffer { - current: 1, - maximum: 5, - actual: 6, - }) - ); - assert!(model.lifecycle(&reference).is_none()); - assert!(model.authenticated_by_slot.is_empty()); - assert!(model.rbc_slots.is_empty()); - } - - #[test] - fn buffered_authenticated_carrier_is_promoted_when_window_opens() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 0); - let future = candidate( - &committee, - 1, - 4, - BlockReference::new_test(1, 3), - vec![ - BlockReference::new_test(0, 3), - BlockReference::new_test(2, 3), - ], - Vec::new(), - 40, - ) - .unwrap(); - let future_ref = future.reference(); - admit(&mut model, future).unwrap(); - assert!(model.lifecycle(&future_ref).unwrap().authenticated); - assert!(!model.lifecycle(&future_ref).unwrap().admitted); - - build_local(&mut model, 1); - for author in [1, 2] { - let (own_prev, weak) = genesis_parents(&committee, author); - let round_one = candidate( - &committee, - author, - 1, - own_prev, - weak, - Vec::new(), - u64::from(author), - ) - .unwrap(); - admit(&mut model, round_one).unwrap(); - } - - assert_eq!(model.local_carrier_round(), 2); - assert!(model.lifecycle(&future_ref).unwrap().admitted); - assert_eq!(model.admitted_reference(1, 4), Some(future_ref)); - - let future_echo = RbcPhaseStatementV1::Echo { target: future_ref }; - for round in 2..=4 { - assert_eq!(model.local_carrier_round(), round); - assert!(!model.pending_phase_batch().contains(&future_echo)); - build_local(&mut model, 100 + u64::from(round)); - assert!(model.pending_phases.contains(&future_echo)); - for author in [2, 3] { - let own_prev = BlockReference::new_test(author, round - 1); - let weak = (0..4) - .filter(|other| *other != author) - .take(2) - .map(|other| BlockReference::new_test(other, round - 1)) - .collect(); - let remote = candidate( - &committee, - author, - round, - own_prev, - weak, - Vec::new(), - u64::from(round) * 10 + u64::from(author), - ) - .unwrap(); - admit(&mut model, remote).unwrap(); - } - assert_eq!(model.local_carrier_round(), round + 1); - } - - assert!(model.pending_phase_batch().contains(&future_echo)); - let round_five = build_local(&mut model, 500); - assert!(round_five.header().phase_batch().contains(&future_echo)); - assert!(!model.pending_phases.contains(&future_echo)); - } - - #[test] - fn poisoned_recipient_still_delivers_without_optimistic_admission() { - let committee = committee(4); - let mut models: Vec<_> = committee - .authorities() - .map(|authority| model(Arc::clone(&committee), authority)) - .collect(); - let round_one: Vec<_> = models - .iter_mut() - .enumerate() - .map(|(index, model)| build_local(model, index as u64)) - .collect(); - let poisoned = round_one[0].reference(); - for carrier in &round_one { - for model in &mut models { - if model.own_authority == carrier.header().author() { - continue; - } - if carrier.reference() == poisoned && model.own_authority == 3 { - model.stage_candidate(carrier.clone()).unwrap(); - } else { - admit(model, carrier.clone()).unwrap(); - } - } - } - assert_ne!(models[3].admitted_reference(0, 1), Some(poisoned)); - for round in 2..=4 { - run_honest_round(&mut models, round); - } - for model in &models { - assert_eq!(model.delivered(0, 1), Some(poisoned)); - } - let poisoned_lifecycle = models[3].lifecycle(&poisoned).unwrap(); - assert!(!poisoned_lifecycle.admitted); - assert!(poisoned_lifecycle.delivered); - } - - #[test] - fn missing_weak_parent_body_does_not_trigger_fetch_or_block_delivery() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (g0, weak0) = genesis_parents(&committee, 0); - let round_one = candidate(&committee, 0, 1, g0, weak0, Vec::new(), 1).unwrap(); - let round_one_ref = round_one.reference(); - force_deliver(&mut model, round_one); - model.mark_data_available(round_one_ref).unwrap(); - - let missing = BlockReference::new_test(1, 1); - let round_two = candidate( - &committee, - 0, - 2, - round_one_ref, - vec![missing, BlockReference::new_test(2, 1)], - Vec::new(), - 2, - ) - .unwrap(); - let round_two_ref = round_two.reference(); - let effects = admit(&mut model, round_two.clone()).unwrap(); - assert!(!effects.iter().any( - |effect| matches!(effect, ModelEffect::NeedCarrier { target, .. } if *target == missing) - )); - force_deliver(&mut model, round_two); - model.mark_data_available(round_two_ref).unwrap(); - assert_eq!(model.prefix_tip(0), Some(round_two_ref)); - } - - #[test] - fn f_missing_weak_parent_bodies_do_not_block_seven_node_delivery() { - let committee = committee(7); - let mut model = model(Arc::clone(&committee), 6); - let (genesis, weak) = genesis_parents(&committee, 0); - let first = candidate(&committee, 0, 1, genesis, weak, Vec::new(), 1).unwrap(); - let first_ref = first.reference(); - force_deliver(&mut model, first); - model.mark_data_available(first_ref).unwrap(); - - let mut known = Vec::new(); - for author in [3, 4] { - let (own_prev, weak) = genesis_parents(&committee, author); - let carrier = candidate( - &committee, - author, - 1, - own_prev, - weak, - Vec::new(), - u64::from(author), - ) - .unwrap(); - known.push(carrier.reference()); - model.stage_candidate(carrier).unwrap(); - } - let missing = [ - BlockReference::new_test(1, 1), - BlockReference::new_test(2, 1), - ]; - let second = candidate( - &committee, - 0, - 2, - first_ref, - vec![missing[0], missing[1], known[0], known[1]], - Vec::new(), - 2, - ) - .unwrap(); - let second_ref = second.reference(); - let effects = admit(&mut model, second.clone()).unwrap(); - assert!(effects.iter().all( - |effect| !matches!(effect, ModelEffect::NeedCarrier { target, .. } if missing.contains(target)) - )); - assert!( - missing - .iter() - .all(|reference| model.lifecycle(reference).is_none()) - ); - - force_deliver(&mut model, second); - model.mark_data_available(second_ref).unwrap(); - assert_eq!(model.prefix_tip(0), Some(second_ref)); - } - - #[test] - fn prefix_rejects_fork_above_closed_tip() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (genesis, weak) = genesis_parents(&committee, 0); - let first = candidate(&committee, 0, 1, genesis, weak, Vec::new(), 1).unwrap(); - let first_ref = first.reference(); - force_deliver(&mut model, first); - model.mark_data_available(first_ref).unwrap(); - - let mut unavailable_fork = first_ref; - unavailable_fork.digest = crate::types::BlockDigest::from([0x88; 32]); - let fork = candidate( - &committee, - 0, - 2, - unavailable_fork, - vec![ - BlockReference::new_test(1, 1), - BlockReference::new_test(2, 1), - ], - Vec::new(), - 2, - ) - .unwrap(); - let fork_ref = fork.reference(); - force_deliver(&mut model, fork); - model.mark_data_available(fork_ref).unwrap(); - assert_eq!(model.prefix_tip(0), Some(first_ref)); - assert!(!model.lifecycle(&fork_ref).unwrap().prefix_closed); - } - - #[test] - fn delayed_data_availability_closes_the_whole_waiting_prefix() { - let committee = committee(4); - let mut model = model(Arc::clone(&committee), 3); - let (genesis, weak) = genesis_parents(&committee, 0); - let first = candidate(&committee, 0, 1, genesis, weak, Vec::new(), 1).unwrap(); - let first_ref = first.reference(); - force_deliver(&mut model, first); - model.mark_data_available(first_ref).unwrap(); - let second = candidate( - &committee, - 0, - 2, - first_ref, - vec![ - BlockReference::new_test(1, 1), - BlockReference::new_test(2, 1), - ], - Vec::new(), - 2, - ) - .unwrap(); - let second_ref = second.reference(); - force_deliver(&mut model, second); - let third = candidate( - &committee, - 0, - 3, - second_ref, - vec![ - BlockReference::new_test(1, 2), - BlockReference::new_test(2, 2), - ], - Vec::new(), - 3, - ) - .unwrap(); - let third_ref = third.reference(); - force_deliver(&mut model, third); - model.mark_data_available(third_ref).unwrap(); - assert_eq!(model.prefix_tip(0), Some(first_ref)); - - let effects = model.mark_data_available(second_ref).unwrap(); - assert_eq!(model.prefix_tip(0), Some(third_ref)); - assert_eq!( - effects - .iter() - .filter(|effect| matches!(effect, ModelEffect::PrefixAdvanced { .. })) - .count(), - 2 - ); - } - - #[test] - fn equal_frontiers_produce_identical_ordered_deltas() { - let committee = committee(4); - let mut left = model(Arc::clone(&committee), 3); - let mut right = model(Arc::clone(&committee), 3); - let carriers: Vec<_> = committee - .authorities() - .map(|author| { - let (own_prev, weak) = genesis_parents(&committee, author); - candidate( - &committee, - author, - 1, - own_prev, - weak, - Vec::new(), - u64::from(author), - ) - .unwrap() - }) - .collect(); - for carrier in &carriers { - force_deliver(&mut left, carrier.clone()); - left.mark_data_available(carrier.reference()).unwrap(); - } - for carrier in carriers.iter().rev() { - force_deliver(&mut right, carrier.clone()); - right.mark_data_available(carrier.reference()).unwrap(); - } - let frontier: Vec<_> = carriers - .iter() - .map(|carrier| Some(carrier.reference())) - .collect(); - let left_delta = left.apply_frontier(&frontier).unwrap(); - let right_delta = right.apply_frontier(&frontier).unwrap(); - assert_eq!(left_delta, right_delta); - assert!(left_delta.windows(2).all(|pair| pair[0] < pair[1])); - assert!(left.apply_frontier(&frontier).unwrap().is_empty()); - } -} diff --git a/crates/starfish-core/src/starfish_rbc_dag/projection.rs b/crates/starfish-core/src/starfish_rbc_dag/projection.rs deleted file mode 100644 index 86fc5274..00000000 --- a/crates/starfish-core/src/starfish_rbc_dag/projection.rs +++ /dev/null @@ -1,1548 +0,0 @@ -// Copyright (c) 2026 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Pure executable model of the certified Starfish projection. -//! -//! This module deliberately has no network, storage, pacemaker, or production -//! consensus integration. It models the boundary at which an RBC-delivered, -//! data-available carrier may contribute its optional consensus vertex, and it -//! evaluates the explicit vote/no-vote evidence committed by those vertices. - -use std::{ - collections::{BTreeMap, BTreeSet}, - error::Error, - fmt, - sync::Arc, -}; - -use crate::{ - committee::Committee, - types::{AuthorityIndex, BlockReference, RoundNumber, Stake}, -}; - -use super::{ - CandidateCarrierV1, ConsensusVertexReference, ConsensusVertexV1, LeaderChoiceV1, - RbcDagCommitteeId, RbcDagProjectionError, carrier_genesis_reference, -}; - -/// An indexed exact carrier-prefix frontier. `None` is the authority's virtual -/// genesis prefix; `Some` always identifies an exact carrier value. -pub type DeliveryFrontierV1 = Vec>; - -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct LeaderSlotV1 { - pub author: AuthorityIndex, - pub round: RoundNumber, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ProjectionDecisionV1 { - DirectCommit { - leader: ConsensusVertexReference, - }, - DirectSkip { - slot: LeaderSlotV1, - }, - IndirectCommit { - leader: ConsensusVertexReference, - anchor: ConsensusVertexReference, - }, - IndirectSkip { - slot: LeaderSlotV1, - anchor: ConsensusVertexReference, - }, - Undecided { - slot: LeaderSlotV1, - }, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum CertifiedProjectionError { - CommitteeMismatch, - UnknownCarrier(BlockReference), - ConflictingDeliveredCarrierSlot { - existing: BlockReference, - conflicting: BlockReference, - }, - MissingConsensusVertex(BlockReference), - InvalidProjectionShape(RbcDagProjectionError), - CarrierNotDelivered(BlockReference), - CarrierDataUnavailable(BlockReference), - CarrierOutsideClosedPrefix(BlockReference), - MissingStrongParent(ConsensusVertexReference), - InvalidGenesisStrongParent(ConsensusVertexReference), - OwnFrontierDoesNotNamePreviousCarrier { - expected: Option, - actual: Option, - }, - FrontierNotClosed { - authority: AuthorityIndex, - reference: BlockReference, - }, - ParentFrontierFork { - authority: AuthorityIndex, - left: Option, - right: Option, - }, - FrontierDoesNotDominateParent { - authority: AuthorityIndex, - required: Option, - actual: Option, - }, - FrontierRegressesCommitted { - authority: AuthorityIndex, - committed: Option, - actual: Option, - }, - StakeOverflow, - InvalidLeaderSlot(LeaderSlotV1), - MultipleCertifiedLeaderValues(LeaderSlotV1), - ConflictingDirectDecision(LeaderSlotV1), - AnchorNotCommitted(ConsensusVertexReference), - AnchorTooEarly { - slot: LeaderSlotV1, - anchor: ConsensusVertexReference, - }, -} - -impl fmt::Display for CertifiedProjectionError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "certified projection error: {self:?}") - } -} - -impl Error for CertifiedProjectionError {} - -#[derive(Clone, Debug)] -struct CarrierState { - candidate: CandidateCarrierV1, - delivered: bool, - data_available: bool, -} - -#[derive(Clone, Debug)] -struct ProjectedVertex { - vertex: ConsensusVertexV1, - effective_frontier: DeliveryFrontierV1, -} - -/// Stateful, deterministic model of the clean certified projection. -/// -/// Staging and cleaning a carrier never projects its optional vertex -/// automatically. Callers explicitly invoke [`Self::try_project`] so tests can -/// observe malformed or unavailable optional metadata without changing carrier -/// admission state. -#[derive(Clone)] -pub struct CertifiedProjectionModel { - committee: Arc, - committee_id: RbcDagCommitteeId, - carriers: BTreeMap, - delivered_slots: BTreeMap<(AuthorityIndex, RoundNumber), BlockReference>, - closed_prefixes: Vec>, - vertices: BTreeMap, - consensus_slots: BTreeMap<(AuthorityIndex, RoundNumber), BTreeSet>, - committed_frontier: DeliveryFrontierV1, - committed_anchors: BTreeSet, -} - -impl CertifiedProjectionModel { - pub fn new(committee: Arc) -> Result { - let committee_id = RbcDagCommitteeId::derive(&committee) - .map_err(|_| CertifiedProjectionError::CommitteeMismatch)?; - let committee_size = committee.len(); - Ok(Self { - committee, - committee_id, - carriers: BTreeMap::new(), - delivered_slots: BTreeMap::new(), - closed_prefixes: vec![Vec::new(); committee_size], - vertices: BTreeMap::new(), - consensus_slots: BTreeMap::new(), - committed_frontier: vec![None; committee_size], - committed_anchors: BTreeSet::new(), - }) - } - - /// Retain a canonical carrier independently of optional-vertex validity. - pub fn stage_carrier( - &mut self, - candidate: CandidateCarrierV1, - ) -> Result<(), CertifiedProjectionError> { - if candidate.committee_id() != self.committee_id { - return Err(CertifiedProjectionError::CommitteeMismatch); - } - self.carriers - .entry(candidate.reference()) - .or_insert(CarrierState { - candidate, - delivered: false, - data_available: false, - }); - Ok(()) - } - - /// Record exact RBC delivery. A second delivered value in one physical - /// author/round slot is rejected rather than resolved by arrival order. - pub fn mark_delivered( - &mut self, - reference: BlockReference, - ) -> Result<(), CertifiedProjectionError> { - let state = self - .carriers - .get(&reference) - .ok_or(CertifiedProjectionError::UnknownCarrier(reference))?; - let slot = ( - state.candidate.header().author(), - state.candidate.header().carrier_round(), - ); - if let Some(existing) = self.delivered_slots.get(&slot) { - if *existing != reference { - return Err(CertifiedProjectionError::ConflictingDeliveredCarrierSlot { - existing: *existing, - conflicting: reference, - }); - } - } - self.delivered_slots.insert(slot, reference); - self.carriers - .get_mut(&reference) - .expect("carrier checked above") - .delivered = true; - self.advance_closed_prefix(slot.0); - Ok(()) - } - - pub fn mark_data_available( - &mut self, - reference: BlockReference, - ) -> Result<(), CertifiedProjectionError> { - let authority = self - .carriers - .get(&reference) - .ok_or(CertifiedProjectionError::UnknownCarrier(reference))? - .candidate - .header() - .author(); - self.carriers - .get_mut(&reference) - .expect("carrier checked above") - .data_available = true; - self.advance_closed_prefix(authority); - Ok(()) - } - - pub fn carrier_is_stored(&self, reference: BlockReference) -> bool { - self.carriers.contains_key(&reference) - } - - pub fn closed_tip(&self, authority: AuthorityIndex) -> Option { - self.closed_prefixes - .get(authority as usize) - .and_then(|prefix| prefix.last()) - .copied() - } - - pub fn is_projected(&self, reference: ConsensusVertexReference) -> bool { - self.vertices.contains_key(&reference) - } - - pub fn slot_values( - &self, - author: AuthorityIndex, - round: RoundNumber, - ) -> Vec { - self.consensus_slots - .get(&(author, round)) - .map(|values| values.iter().copied().collect()) - .unwrap_or_default() - } - - pub fn effective_frontier( - &self, - reference: ConsensusVertexReference, - ) -> Option<&[Option]> { - self.vertices - .get(&reference) - .map(|projected| projected.effective_frontier.as_slice()) - } - - pub fn leader_choice(&self, reference: ConsensusVertexReference) -> Option { - self.vertices - .get(&reference) - .map(|projected| projected.vertex.leader_choice()) - } - - /// Project one optional consensus vertex if every stateful eligibility - /// condition holds. Failure leaves the enclosing carrier untouched. - pub fn try_project( - &mut self, - carrier_reference: BlockReference, - ) -> Result { - let state = self - .carriers - .get(&carrier_reference) - .ok_or(CertifiedProjectionError::UnknownCarrier(carrier_reference))?; - let vertex = state.candidate.header().consensus_vertex().cloned().ok_or( - CertifiedProjectionError::MissingConsensusVertex(carrier_reference), - )?; - let vertex_reference = - ConsensusVertexReference::new(carrier_reference, vertex.consensus_round()); - if self.vertices.contains_key(&vertex_reference) { - return Ok(vertex_reference); - } - - state - .candidate - .validate_consensus_vertex(&self.committee) - .map_err(CertifiedProjectionError::InvalidProjectionShape)?; - if !state.delivered { - return Err(CertifiedProjectionError::CarrierNotDelivered( - carrier_reference, - )); - } - if !state.data_available { - return Err(CertifiedProjectionError::CarrierDataUnavailable( - carrier_reference, - )); - } - if !self.is_on_closed_prefix(carrier_reference) { - return Err(CertifiedProjectionError::CarrierOutsideClosedPrefix( - carrier_reference, - )); - } - - let author = carrier_reference.authority; - let own_previous = state.candidate.header().own_prev(); - let expected_author_frontier = (own_previous.round != 0).then_some(own_previous); - let actual_author_frontier = vertex.delivery_frontier()[author as usize]; - if actual_author_frontier != expected_author_frontier { - return Err( - CertifiedProjectionError::OwnFrontierDoesNotNamePreviousCarrier { - expected: expected_author_frontier, - actual: actual_author_frontier, - }, - ); - } - - self.ensure_frontier_closed(vertex.delivery_frontier())?; - - let mut parent_frontiers = Vec::with_capacity(vertex.strong_parents().len()); - for parent in vertex.strong_parents() { - if parent.consensus_round() == 0 { - if parent.carrier() != carrier_genesis_reference(parent.author()) { - return Err(CertifiedProjectionError::InvalidGenesisStrongParent( - *parent, - )); - } - parent_frontiers.push(vec![None; self.committee.len()]); - continue; - } - let projected = self - .vertices - .get(parent) - .ok_or(CertifiedProjectionError::MissingStrongParent(*parent))?; - parent_frontiers.push(projected.effective_frontier.clone()); - } - let parent_join = self.join_frontiers(&parent_frontiers)?; - self.ensure_dominates_parent(vertex.delivery_frontier(), &parent_join)?; - - let mut effective_frontier = vertex.delivery_frontier().to_vec(); - effective_frontier[author as usize] = Some(carrier_reference); - self.vertices.insert( - vertex_reference, - ProjectedVertex { - vertex, - effective_frontier, - }, - ); - self.consensus_slots - .entry((author, vertex_reference.consensus_round())) - .or_default() - .insert(vertex_reference); - Ok(vertex_reference) - } - - /// The scheduled leader slot for a consensus round. - pub fn leader_slot(&self, round: RoundNumber) -> LeaderSlotV1 { - LeaderSlotV1 { - author: self.committee.elect_leader(round), - round, - } - } - - /// Stake of distinct projected voter authors that explicitly vote for this - /// exact leader value. Equivocations by one author count once. - pub fn vote_stake( - &self, - leader: ConsensusVertexReference, - ) -> Result { - let slot = LeaderSlotV1 { - author: leader.author(), - round: leader.consensus_round(), - }; - self.validate_leader_slot(slot)?; - self.stake_of_authors(self.voter_authors( - slot, - |choice| matches!(choice, LeaderChoiceV1::Vote { leader: voted } if voted == leader), - )) - } - - /// Evaluate the explicit direct Starfish patterns over clean projected - /// vertices only. - pub fn direct_decision( - &self, - slot: LeaderSlotV1, - ) -> Result { - self.validate_leader_slot(slot)?; - let candidates = self.slot_values(slot.author, slot.round); - let mut committed = Vec::new(); - for candidate in &candidates { - let certifier_authors = self.certifier_authors(*candidate); - if self.stake_of_authors(certifier_authors)? >= self.committee.quorum_threshold() { - committed.push(*candidate); - } - } - if committed.len() > 1 { - return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues( - slot, - )); - } - - let all_choice_authors = self.voter_authors(slot, |_| true); - let enough_choices = - self.stake_of_authors(all_choice_authors)? >= self.committee.quorum_threshold(); - let skip = enough_choices - && candidates.iter().all(|candidate| { - self.stake_of_authors(self.voter_authors(slot, |choice| match choice { - LeaderChoiceV1::Vote { leader } => leader != *candidate, - LeaderChoiceV1::NoVote { .. } => true, - })) - .is_ok_and(|stake| stake >= self.committee.quorum_threshold()) - }); - - match (committed.pop(), skip) { - (Some(_), true) => Err(CertifiedProjectionError::ConflictingDirectDecision(slot)), - (Some(leader), false) => Ok(ProjectionDecisionV1::DirectCommit { leader }), - (None, true) => Ok(ProjectionDecisionV1::DirectSkip { slot }), - (None, false) => Ok(ProjectionDecisionV1::Undecided { slot }), - } - } - - /// Record an externally selected committed anchor while enforcing exact - /// componentwise frontier monotonicity. The runtime committer remains out - /// of scope for this model. - pub fn record_committed_anchor( - &mut self, - anchor: ConsensusVertexReference, - ) -> Result<(), CertifiedProjectionError> { - let projected = self - .vertices - .get(&anchor) - .ok_or(CertifiedProjectionError::MissingStrongParent(anchor))?; - let frontier = projected.effective_frontier.clone(); - self.ensure_dominates_committed(&frontier)?; - self.committed_frontier = frontier; - self.committed_anchors.insert(anchor); - Ok(()) - } - - /// Decide an older leader from a later committed anchor. A reachable - /// certifying-round vertex with a QC yields commit; absence yields skip. - pub fn indirect_decision( - &self, - slot: LeaderSlotV1, - anchor: ConsensusVertexReference, - ) -> Result { - self.validate_leader_slot(slot)?; - if !self.committed_anchors.contains(&anchor) { - return Err(CertifiedProjectionError::AnchorNotCommitted(anchor)); - } - let minimum_anchor_round = slot.round.saturating_add(3); - if anchor.consensus_round() < minimum_anchor_round { - return Err(CertifiedProjectionError::AnchorTooEarly { slot, anchor }); - } - let certifying_round = slot.round.saturating_add(2); - let reachable = self.reachable_at_round(anchor, certifying_round); - let mut certified = Vec::new(); - for candidate in self.slot_values(slot.author, slot.round) { - if reachable - .iter() - .any(|certifier| self.is_certificate(*certifier, candidate)) - { - certified.push(candidate); - } - } - if certified.len() > 1 { - return Err(CertifiedProjectionError::MultipleCertifiedLeaderValues( - slot, - )); - } - Ok(match certified.pop() { - Some(leader) => ProjectionDecisionV1::IndirectCommit { leader, anchor }, - None => ProjectionDecisionV1::IndirectSkip { slot, anchor }, - }) - } - - fn advance_closed_prefix(&mut self, authority: AuthorityIndex) { - let index = authority as usize; - loop { - let next_round = self.closed_prefixes[index].len() as RoundNumber + 1; - let Some(reference) = self.delivered_slots.get(&(authority, next_round)).copied() - else { - break; - }; - let Some(state) = self.carriers.get(&reference) else { - break; - }; - if !state.delivered || !state.data_available { - break; - } - let expected_previous = self.closed_prefixes[index] - .last() - .copied() - .unwrap_or_else(|| carrier_genesis_reference(authority)); - if state.candidate.header().own_prev() != expected_previous { - break; - } - self.closed_prefixes[index].push(reference); - } - } - - fn is_on_closed_prefix(&self, reference: BlockReference) -> bool { - self.closed_tip(reference.authority) - .is_some_and(|tip| self.is_exact_extension(Some(reference), Some(tip))) - } - - fn ensure_frontier_closed( - &self, - frontier: &[Option], - ) -> Result<(), CertifiedProjectionError> { - for (index, entry) in frontier.iter().copied().enumerate() { - let Some(reference) = entry else { - continue; - }; - let authority = index as AuthorityIndex; - let closed_tip = self.closed_tip(authority); - if !self.is_exact_extension(Some(reference), closed_tip) { - return Err(CertifiedProjectionError::FrontierNotClosed { - authority, - reference, - }); - } - } - Ok(()) - } - - fn join_frontiers( - &self, - frontiers: &[DeliveryFrontierV1], - ) -> Result { - let mut joined = vec![None; self.committee.len()]; - for frontier in frontiers { - for (index, right) in frontier.iter().copied().enumerate() { - let left = joined[index]; - if self.is_exact_extension(left, right) { - joined[index] = right; - } else if !self.is_exact_extension(right, left) { - return Err(CertifiedProjectionError::ParentFrontierFork { - authority: index as AuthorityIndex, - left, - right, - }); - } - } - } - Ok(joined) - } - - fn ensure_dominates_parent( - &self, - frontier: &[Option], - required: &[Option], - ) -> Result<(), CertifiedProjectionError> { - for (index, (required, actual)) in required - .iter() - .copied() - .zip(frontier.iter().copied()) - .enumerate() - { - if !self.is_exact_extension(required, actual) { - return Err(CertifiedProjectionError::FrontierDoesNotDominateParent { - authority: index as AuthorityIndex, - required, - actual, - }); - } - } - Ok(()) - } - - fn ensure_dominates_committed( - &self, - frontier: &[Option], - ) -> Result<(), CertifiedProjectionError> { - for (index, (committed, actual)) in self - .committed_frontier - .iter() - .copied() - .zip(frontier.iter().copied()) - .enumerate() - { - if !self.is_exact_extension(committed, actual) { - return Err(CertifiedProjectionError::FrontierRegressesCommitted { - authority: index as AuthorityIndex, - committed, - actual, - }); - } - } - Ok(()) - } - - /// True iff `descendant` is the same exact prefix tip as `base`, or an - /// exact self-chain extension whose intermediate carrier headers are known. - fn is_exact_extension( - &self, - base: Option, - descendant: Option, - ) -> bool { - let Some(mut cursor) = descendant else { - return base.is_none(); - }; - let authority = cursor.authority; - if base.is_some_and(|base| base.authority != authority) { - return false; - } - let base_round = base.map_or(0, |reference| reference.round); - if cursor.round < base_round { - return false; - } - while cursor.round > base_round { - let Some(state) = self.carriers.get(&cursor) else { - return false; - }; - if state.candidate.header().author() != authority - || state.candidate.reference() != cursor - { - return false; - } - cursor = state.candidate.header().own_prev(); - } - match base { - Some(reference) => cursor == reference, - None => cursor == carrier_genesis_reference(authority), - } - } - - fn validate_leader_slot(&self, slot: LeaderSlotV1) -> Result<(), CertifiedProjectionError> { - if slot.round == 0 || self.committee.elect_leader(slot.round) != slot.author { - return Err(CertifiedProjectionError::InvalidLeaderSlot(slot)); - } - Ok(()) - } - - fn vertices_at_round( - &self, - round: RoundNumber, - ) -> impl Iterator { - self.vertices - .iter() - .filter(move |(reference, _)| reference.consensus_round() == round) - .map(|(reference, projected)| (*reference, projected)) - } - - fn voter_authors( - &self, - slot: LeaderSlotV1, - predicate: impl Fn(LeaderChoiceV1) -> bool, - ) -> BTreeSet { - self.vertices_at_round(slot.round.saturating_add(1)) - .filter_map(|(reference, projected)| { - let choice = projected.vertex.leader_choice(); - let belongs_to_slot = match choice { - LeaderChoiceV1::Vote { leader } => { - leader.author() == slot.author && leader.consensus_round() == slot.round - } - LeaderChoiceV1::NoVote { - leader_author, - leader_round, - } => leader_author == slot.author && leader_round == slot.round, - }; - (belongs_to_slot && predicate(choice)).then_some(reference.author()) - }) - .collect() - } - - fn certifier_authors(&self, leader: ConsensusVertexReference) -> BTreeSet { - self.vertices_at_round(leader.consensus_round().saturating_add(2)) - .filter_map(|(reference, _)| { - self.is_certificate(reference, leader) - .then_some(reference.author()) - }) - .collect() - } - - fn is_certificate( - &self, - certifier: ConsensusVertexReference, - leader: ConsensusVertexReference, - ) -> bool { - let Some(projected) = self.vertices.get(&certifier) else { - return false; - }; - let voter_authors: BTreeSet<_> = projected - .vertex - .strong_parents() - .iter() - .filter_map(|parent| { - self.vertices.get(parent).and_then(|voter| { - matches!( - voter.vertex.leader_choice(), - LeaderChoiceV1::Vote { leader: voted } if voted == leader - ) - .then_some(parent.author()) - }) - }) - .collect(); - self.stake_of_authors(voter_authors) - .is_ok_and(|stake| stake >= self.committee.quorum_threshold()) - } - - fn stake_of_authors( - &self, - authors: BTreeSet, - ) -> Result { - authors.into_iter().try_fold(0u64, |stake, author| { - let author_stake = self - .committee - .get_stake(author) - .ok_or(CertifiedProjectionError::StakeOverflow)?; - stake - .checked_add(author_stake) - .ok_or(CertifiedProjectionError::StakeOverflow) - }) - } - - fn reachable_at_round( - &self, - anchor: ConsensusVertexReference, - target_round: RoundNumber, - ) -> BTreeSet { - let mut result = BTreeSet::new(); - let mut pending = vec![anchor]; - let mut seen = BTreeSet::new(); - while let Some(reference) = pending.pop() { - if !seen.insert(reference) || reference.consensus_round() < target_round { - continue; - } - if reference.consensus_round() == target_round { - result.insert(reference); - continue; - } - if let Some(projected) = self.vertices.get(&reference) { - pending.extend(projected.vertex.strong_parents().iter().copied()); - } - } - result - } - - #[cfg(test)] - fn inject_projected_for_test( - &mut self, - reference: ConsensusVertexReference, - strong_parents: Vec, - leader_choice: LeaderChoiceV1, - ) { - let vertex = ConsensusVertexV1::new( - reference.consensus_round(), - strong_parents, - vec![None; self.committee.len()], - leader_choice, - ); - self.vertices.insert( - reference, - ProjectedVertex { - vertex, - effective_frontier: vec![None; self.committee.len()], - }, - ); - self.consensus_slots - .entry((reference.author(), reference.consensus_round())) - .or_default() - .insert(reference); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - crypto::TransactionsCommitment, - starfish_rbc_dag::{CarrierHeaderV1Args, LeaderChoiceV1}, - types::BlockDigest, - }; - - fn reference(authority: AuthorityIndex, round: RoundNumber, marker: u8) -> BlockReference { - BlockReference { - authority, - round, - digest: BlockDigest::from([marker; 32]), - } - } - - fn consensus_reference( - authority: AuthorityIndex, - round: RoundNumber, - marker: u8, - ) -> ConsensusVertexReference { - ConsensusVertexReference::new(reference(authority, round + 20, marker), round) - } - - fn previous_carriers(committee: &Committee, round: RoundNumber) -> Vec { - committee - .authorities() - .map(|authority| { - if round == 0 { - carrier_genesis_reference(authority) - } else { - reference(authority, round, authority as u8 + 0x80) - } - }) - .collect() - } - - fn candidate( - committee: &Committee, - author: AuthorityIndex, - carrier_round: RoundNumber, - previous: &[BlockReference], - vertex: Option, - marker: u8, - ) -> CandidateCarrierV1 { - let weak_parents = previous - .iter() - .copied() - .filter(|parent| parent.authority != author) - .collect(); - CandidateCarrierV1::try_new( - CarrierHeaderV1Args { - author, - carrier_round, - own_prev: previous[author as usize], - weak_parents, - transactions_commitment: TransactionsCommitment::from_bytes([marker; 32]), - data_acknowledgments: Vec::new(), - phase_batch: Vec::new(), - consensus_vertex: vertex, - creation_time_ns: marker as u64, - }, - committee, - ) - .unwrap() - } - - fn clean( - model: &mut CertifiedProjectionModel, - candidate: CandidateCarrierV1, - ) -> BlockReference { - let reference = candidate.reference(); - model.stage_carrier(candidate).unwrap(); - model.mark_delivered(reference).unwrap(); - model.mark_data_available(reference).unwrap(); - reference - } - - fn first_consensus_round( - model: &mut CertifiedProjectionModel, - ) -> (Vec, Vec) { - let previous = previous_carriers(&model.committee, 0); - let strong_parents: Vec<_> = model - .committee - .authorities() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - let leader = strong_parents[0]; - let mut carriers = Vec::new(); - let mut vertices = Vec::new(); - let authors: Vec<_> = model.committee.authorities().collect(); - for author in authors { - let vertex = ConsensusVertexV1::new( - 1, - strong_parents.clone(), - vec![None; model.committee.len()], - LeaderChoiceV1::Vote { leader }, - ); - let carrier = candidate( - &model.committee, - author, - 1, - &previous, - Some(vertex), - 0x10 + author as u8, - ); - let carrier_reference = clean(model, carrier); - let vertex_reference = model.try_project(carrier_reference).unwrap(); - carriers.push(carrier_reference); - vertices.push(vertex_reference); - } - (carriers, vertices) - } - - fn second_round_candidate( - model: &CertifiedProjectionModel, - author: AuthorityIndex, - previous: &[BlockReference], - parents: Vec, - frontier: DeliveryFrontierV1, - marker: u8, - ) -> CandidateCarrierV1 { - let leader = parents - .iter() - .find(|parent| parent.author() == model.committee.elect_leader(1)) - .copied() - .unwrap(); - candidate( - &model.committee, - author, - 2, - previous, - Some(ConsensusVertexV1::new( - 2, - parents, - frontier, - LeaderChoiceV1::Vote { leader }, - )), - marker, - ) - } - - #[test] - fn effective_frontier_includes_every_enclosing_strong_parent() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let (carriers, parents) = first_consensus_round(&mut model); - - let child = second_round_candidate( - &model, - 0, - &carriers, - parents, - carriers.iter().copied().map(Some).collect(), - 0x30, - ); - let child_carrier = clean(&mut model, child); - let child_vertex = model.try_project(child_carrier).unwrap(); - let effective = model.effective_frontier(child_vertex).unwrap(); - assert_eq!(effective[0], Some(child_carrier)); - let expected: Vec<_> = carriers[1..].iter().copied().map(Some).collect(); - assert_eq!(&effective[1..], expected.as_slice()); - } - - #[test] - fn omission_and_same_round_fork_do_not_pass_frontier_checks() { - let committee = Committee::new_test(vec![1; 4]); - let mut omission_model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let (carriers, parents) = first_consensus_round(&mut omission_model); - let mut omitted = carriers.iter().copied().map(Some).collect::>(); - omitted[2] = None; - let child = second_round_candidate(&omission_model, 0, &carriers, parents, omitted, 0x31); - let child_reference = clean(&mut omission_model, child); - assert!(matches!( - omission_model.try_project(child_reference), - Err(CertifiedProjectionError::FrontierDoesNotDominateParent { authority: 2, .. }) - )); - assert!(omission_model.carrier_is_stored(child_reference)); - - let mut fork_model = CertifiedProjectionModel::new(committee).unwrap(); - let (carriers, parents) = first_consensus_round(&mut fork_model); - let mut forked = carriers.iter().copied().map(Some).collect::>(); - forked[2] = Some(reference(2, 1, 0xEE)); - let child = second_round_candidate(&fork_model, 0, &carriers, parents, forked, 0x32); - let child_reference = clean(&mut fork_model, child); - assert!(matches!( - fork_model.try_project(child_reference), - Err(CertifiedProjectionError::FrontierNotClosed { authority: 2, .. }) - )); - } - - #[test] - fn exact_strong_parent_lookup_and_parent_fork_are_enforced() { - let committee = Committee::new_test(vec![1; 4]); - let mut exact_model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let (carriers, mut parents) = first_consensus_round(&mut exact_model); - parents[2] = ConsensusVertexReference::new(reference(2, 1, 0xEF), 1); - let child = second_round_candidate( - &exact_model, - 0, - &carriers, - parents, - carriers.iter().copied().map(Some).collect(), - 0x33, - ); - let child_reference = clean(&mut exact_model, child); - assert!(matches!( - exact_model.try_project(child_reference), - Err(CertifiedProjectionError::MissingStrongParent(parent)) - if parent.author() == 2 - )); - - let mut fork_model = CertifiedProjectionModel::new(committee).unwrap(); - let genesis = previous_carriers(&fork_model.committee, 0); - let base = candidate(&fork_model.committee, 0, 1, &genesis, None, 0x34); - let base_reference = clean(&mut fork_model, base); - let parent_references: Vec<_> = (0..3) - .map(|authority| consensus_reference(authority, 1, 0x40 + authority as u8)) - .collect(); - for parent in &parent_references { - fork_model.inject_projected_for_test( - *parent, - Vec::new(), - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }, - ); - } - fork_model - .vertices - .get_mut(&parent_references[0]) - .unwrap() - .effective_frontier[3] = Some(reference(3, 1, 0xA1)); - fork_model - .vertices - .get_mut(&parent_references[1]) - .unwrap() - .effective_frontier[3] = Some(reference(3, 1, 0xA2)); - let previous = vec![ - base_reference, - reference(1, 1, 0xB1), - reference(2, 1, 0xB2), - reference(3, 1, 0xB3), - ]; - let child = second_round_candidate( - &fork_model, - 0, - &previous, - parent_references, - vec![Some(base_reference), None, None, None], - 0x35, - ); - let child_reference = clean(&mut fork_model, child); - assert!(matches!( - fork_model.try_project(child_reference), - Err(CertifiedProjectionError::ParentFrontierFork { authority: 3, .. }) - )); - } - - #[test] - fn late_vertex_remains_visible_but_cannot_become_a_regressing_anchor() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (carriers, parents) = first_consensus_round(&mut model); - let anchor = second_round_candidate( - &model, - 0, - &carriers, - parents.clone(), - carriers.iter().copied().map(Some).collect(), - 0x36, - ); - let anchor_carrier = clean(&mut model, anchor); - let anchor_vertex = model.try_project(anchor_carrier).unwrap(); - model.record_committed_anchor(anchor_vertex).unwrap(); - - let regressing = second_round_candidate( - &model, - 2, - &carriers, - parents, - carriers.iter().copied().map(Some).collect(), - 0x37, - ); - let regressing_carrier = clean(&mut model, regressing); - let regressing_vertex = model.try_project(regressing_carrier).unwrap(); - assert!(model.is_projected(regressing_vertex)); - assert!(matches!( - model.record_committed_anchor(regressing_vertex), - Err(CertifiedProjectionError::FrontierRegressesCommitted { authority: 0, .. }) - )); - } - - #[test] - fn conflicting_consensus_values_in_one_slot_remain_visible() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (carriers, _) = first_consensus_round(&mut model); - let genesis_parents: Vec<_> = model - .committee - .authorities() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - let conflicting = candidate( - &model.committee, - 0, - 2, - &carriers, - Some(ConsensusVertexV1::new( - 1, - genesis_parents.clone(), - carriers.iter().copied().map(Some).collect(), - LeaderChoiceV1::Vote { - leader: genesis_parents[0], - }, - )), - 0x38, - ); - let conflicting_carrier = clean(&mut model, conflicting); - model.try_project(conflicting_carrier).unwrap(); - - let values = model.slot_values(0, 1); - assert_eq!(values.len(), 2); - assert_ne!(values[0], values[1]); - } - - #[test] - fn objective_vote_and_no_vote_shapes_are_both_projectable() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let previous = previous_carriers(&committee, 0); - let genesis: Vec<_> = committee - .authorities() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - let vote = candidate( - &committee, - 1, - 1, - &previous, - Some(ConsensusVertexV1::new( - 1, - vec![genesis[0], genesis[1], genesis[2]], - vec![None; 4], - LeaderChoiceV1::Vote { leader: genesis[0] }, - )), - 0x39, - ); - let vote_carrier = clean(&mut model, vote); - let vote_reference = model.try_project(vote_carrier).unwrap(); - let no_vote = candidate( - &committee, - 3, - 1, - &previous, - Some(ConsensusVertexV1::new( - 1, - vec![genesis[1], genesis[2], genesis[3]], - vec![None; 4], - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }, - )), - 0x3A, - ); - let no_vote_carrier = clean(&mut model, no_vote); - let no_vote_reference = model.try_project(no_vote_carrier).unwrap(); - - assert_eq!( - model.leader_choice(vote_reference), - Some(LeaderChoiceV1::Vote { leader: genesis[0] }) - ); - assert!(matches!( - model.leader_choice(no_vote_reference), - Some(LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0 - }) - )); - } - - #[test] - fn dirty_or_malformed_optional_vertex_never_enters_projection() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let previous = previous_carriers(&committee, 0); - let genesis: Vec<_> = committee - .authorities() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - let dirty = candidate( - &committee, - 0, - 1, - &previous, - Some(ConsensusVertexV1::new( - 1, - vec![genesis[0], genesis[1], genesis[2]], - vec![None; 4], - LeaderChoiceV1::Vote { leader: genesis[0] }, - )), - 0x3B, - ); - let dirty_carrier = dirty.reference(); - model.stage_carrier(dirty).unwrap(); - assert_eq!( - model.try_project(dirty_carrier), - Err(CertifiedProjectionError::CarrierNotDelivered(dirty_carrier)) - ); - assert!(model.carrier_is_stored(dirty_carrier)); - assert!(model.slot_values(0, 1).is_empty()); - - let malformed = candidate( - &committee, - 1, - 1, - &previous, - Some(ConsensusVertexV1::new( - 1, - vec![genesis[1]], - vec![None; 4], - LeaderChoiceV1::NoVote { - leader_author: 0, - leader_round: 0, - }, - )), - 0x3C, - ); - let malformed_carrier = clean(&mut model, malformed); - assert!(matches!( - model.try_project(malformed_carrier), - Err(CertifiedProjectionError::InvalidProjectionShape( - RbcDagProjectionError::InvalidStrongParentThreshold - )) - )); - assert!(model.carrier_is_stored(malformed_carrier)); - assert!(model.slot_values(1, 1).is_empty()); - } - - #[test] - fn missing_weak_parent_bodies_do_not_block_consensus_projection() { - let committee = Committee::new_test(vec![1; 7]); - let mut model = CertifiedProjectionModel::new(Arc::clone(&committee)).unwrap(); - let genesis = previous_carriers(&committee, 0); - let base = candidate(&committee, 0, 1, &genesis, None, 0x3D); - let base_reference = clean(&mut model, base); - - let mut previous = previous_carriers(&committee, 1); - previous[0] = base_reference; - let missing: Vec<_> = previous[1..=2].to_vec(); - let strong_parents: Vec<_> = committee - .authorities() - .map(|authority| ConsensusVertexReference::new(carrier_genesis_reference(authority), 0)) - .collect(); - let leader = strong_parents - .iter() - .find(|parent| parent.author() == committee.elect_leader(0)) - .copied() - .unwrap(); - let mut frontier = vec![None; committee.len()]; - frontier[0] = Some(base_reference); - let outer = candidate( - &committee, - 0, - 2, - &previous, - Some(ConsensusVertexV1::new( - 1, - strong_parents, - frontier, - LeaderChoiceV1::Vote { leader }, - )), - 0x3E, - ); - let outer_reference = clean(&mut model, outer); - - assert!( - missing - .iter() - .all(|reference| !model.carrier_is_stored(*reference)) - ); - assert!(model.try_project(outer_reference).is_ok()); - } - - fn project_complete_round( - model: &mut CertifiedProjectionModel, - consensus_round: RoundNumber, - previous_carriers: &[BlockReference], - parent_sets: &[Vec], - choices: &[LeaderChoiceV1], - marker_base: u8, - ) -> (Vec, Vec) { - assert_eq!(parent_sets.len(), model.committee.len()); - assert_eq!(choices.len(), model.committee.len()); - let carrier_round = previous_carriers[0].round + 1; - let frontier: DeliveryFrontierV1 = previous_carriers.iter().copied().map(Some).collect(); - let mut carriers = Vec::with_capacity(model.committee.len()); - let mut vertices = Vec::with_capacity(model.committee.len()); - for author in 0..model.committee.len() as AuthorityIndex { - let carrier = candidate( - &model.committee, - author, - carrier_round, - previous_carriers, - Some(ConsensusVertexV1::new( - consensus_round, - parent_sets[author as usize].clone(), - frontier.clone(), - choices[author as usize], - )), - marker_base + author as u8, - ); - let carrier_reference = clean(model, carrier); - let vertex_reference = model.try_project(carrier_reference).unwrap(); - carriers.push(carrier_reference); - vertices.push(vertex_reference); - } - (carriers, vertices) - } - - #[test] - fn direct_commit_uses_clean_projected_voters_and_certifiers() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); - let slot = model.leader_slot(1); - let leader = round_one_vertices[slot.author as usize]; - - let voter_parents = vec![round_one_vertices.clone(); 4]; - let voter_choices = vec![LeaderChoiceV1::Vote { leader }; 4]; - let (round_two_carriers, round_two_vertices) = project_complete_round( - &mut model, - 2, - &round_one_carriers, - &voter_parents, - &voter_choices, - 0x50, - ); - assert_eq!(model.vote_stake(leader).unwrap(), 4); - - let round_two_leader = round_two_vertices[model.committee.elect_leader(2) as usize]; - let certifier_parents = vec![round_two_vertices; 4]; - let certifier_choices = vec![ - LeaderChoiceV1::Vote { - leader: round_two_leader, - }; - 4 - ]; - project_complete_round( - &mut model, - 3, - &round_two_carriers, - &certifier_parents, - &certifier_choices, - 0x60, - ); - - assert_eq!( - model.direct_decision(slot).unwrap(), - ProjectionDecisionV1::DirectCommit { leader } - ); - } - - #[test] - fn direct_skip_uses_clean_projected_explicit_negative_choices() { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); - let slot = model.leader_slot(1); - let leader = round_one_vertices[slot.author as usize]; - let negative_parents = vec![ - round_one_vertices[0], - round_one_vertices[2], - round_one_vertices[3], - ]; - let voter_parents = vec![ - negative_parents.clone(), - vec![ - round_one_vertices[0], - round_one_vertices[1], - round_one_vertices[2], - ], - negative_parents.clone(), - negative_parents, - ]; - let no_vote = LeaderChoiceV1::NoVote { - leader_author: slot.author, - leader_round: slot.round, - }; - let voter_choices = vec![no_vote, LeaderChoiceV1::Vote { leader }, no_vote, no_vote]; - project_complete_round( - &mut model, - 2, - &round_one_carriers, - &voter_parents, - &voter_choices, - 0x70, - ); - - assert_eq!( - model.direct_decision(slot).unwrap(), - ProjectionDecisionV1::DirectSkip { slot } - ); - } - - fn indirect_graph( - include_certificate: bool, - ) -> ( - CertifiedProjectionModel, - LeaderSlotV1, - ConsensusVertexReference, - ConsensusVertexReference, - ) { - let committee = Committee::new_test(vec![1; 4]); - let mut model = CertifiedProjectionModel::new(committee).unwrap(); - let (round_one_carriers, round_one_vertices) = first_consensus_round(&mut model); - let slot = model.leader_slot(1); - let leader = round_one_vertices[slot.author as usize]; - - let voter_parent_sets = vec![ - vec![ - round_one_vertices[0], - round_one_vertices[1], - round_one_vertices[2], - ], - vec![ - round_one_vertices[0], - round_one_vertices[1], - round_one_vertices[2], - ], - vec![ - round_one_vertices[0], - round_one_vertices[1], - round_one_vertices[2], - ], - vec![ - round_one_vertices[0], - round_one_vertices[2], - round_one_vertices[3], - ], - ]; - let no_vote_round_one = LeaderChoiceV1::NoVote { - leader_author: slot.author, - leader_round: slot.round, - }; - let voter_choices = vec![ - LeaderChoiceV1::Vote { leader }, - LeaderChoiceV1::Vote { leader }, - LeaderChoiceV1::Vote { leader }, - no_vote_round_one, - ]; - let (round_two_carriers, round_two_vertices) = project_complete_round( - &mut model, - 2, - &round_one_carriers, - &voter_parent_sets, - &voter_choices, - 0x80, - ); - - let certifier_parent_sets = vec![ - vec![ - round_two_vertices[0], - round_two_vertices[1], - round_two_vertices[2], - ], - vec![ - round_two_vertices[0], - round_two_vertices[1], - round_two_vertices[3], - ], - vec![ - round_two_vertices[0], - round_two_vertices[2], - round_two_vertices[3], - ], - vec![ - round_two_vertices[1], - round_two_vertices[2], - round_two_vertices[3], - ], - ]; - let round_two_leader = round_two_vertices[model.committee.elect_leader(2) as usize]; - let no_vote_round_two = LeaderChoiceV1::NoVote { - leader_author: model.committee.elect_leader(2), - leader_round: 2, - }; - let certifier_choices = vec![ - LeaderChoiceV1::Vote { - leader: round_two_leader, - }, - no_vote_round_two, - LeaderChoiceV1::Vote { - leader: round_two_leader, - }, - LeaderChoiceV1::Vote { - leader: round_two_leader, - }, - ]; - let (round_three_carriers, round_three_vertices) = project_complete_round( - &mut model, - 3, - &round_two_carriers, - &certifier_parent_sets, - &certifier_choices, - 0x90, - ); - assert_eq!( - model.direct_decision(slot).unwrap(), - ProjectionDecisionV1::Undecided { slot } - ); - - let (anchor_author, anchor_parents, anchor_choice) = if include_certificate { - ( - 0, - round_three_vertices[..3].to_vec(), - LeaderChoiceV1::NoVote { - leader_author: model.committee.elect_leader(3), - leader_round: 3, - }, - ) - } else { - ( - 3, - round_three_vertices[1..].to_vec(), - LeaderChoiceV1::Vote { - leader: round_three_vertices[3], - }, - ) - }; - let anchor_carrier = candidate( - &model.committee, - anchor_author, - 4, - &round_three_carriers, - Some(ConsensusVertexV1::new( - 4, - anchor_parents, - round_three_carriers.iter().copied().map(Some).collect(), - anchor_choice, - )), - if include_certificate { 0xA0 } else { 0xA1 }, - ); - let anchor_carrier = clean(&mut model, anchor_carrier); - let anchor = model.try_project(anchor_carrier).unwrap(); - model.record_committed_anchor(anchor).unwrap(); - (model, slot, leader, anchor) - } - - #[test] - fn later_committed_anchor_drives_indirect_commit_or_skip() { - let (commit_model, slot, leader, commit_anchor) = indirect_graph(true); - assert_eq!( - commit_model.indirect_decision(slot, commit_anchor).unwrap(), - ProjectionDecisionV1::IndirectCommit { - leader, - anchor: commit_anchor, - } - ); - - let (skip_model, slot, _, skip_anchor) = indirect_graph(false); - assert_eq!( - skip_model.indirect_decision(slot, skip_anchor).unwrap(), - ProjectionDecisionV1::IndirectSkip { - slot, - anchor: skip_anchor, - } - ); - } -} diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md index 385a9c54..970dc964 100644 --- a/docs/starfish-rbc-dag-protocol.md +++ b/docs/starfish-rbc-dag-protocol.md @@ -41,17 +41,10 @@ remain selectable outer-authentication baselines; changing that selector does no embedded RBC or consensus rules. This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish -commit rules already exist. The isolated milestone-two implementation now provides a canonical -codec plus deterministic carrier/RBC, certified-projection, decision, and crash-journal models. -Those models are not runtime integration or a proof: the composition through two clocks, two -logical projections, and frontier-based payload ordering still requires shadow execution, -additional adversarial testing, and a safety/liveness argument. Until those obligations are -discharged, `starfish-rbc-dag` must be described as an experimental reference model rather than a -proven signature-free Starfish variant. - -The milestone-two model accepts `DataAvailable` as a trusted input from the existing verified -Reed-Solomon/reconstruction layer. It models the resulting prefix and ordering transitions, but not -payload reconstruction or the runtime transition from delivered acknowledgments to that input. +commit rules already exist, but their composition through two clocks, two logical projections, and +frontier-based payload ordering still requires an executable model, adversarial tests, and a proof. +Until those obligations are discharged, `starfish-rbc-dag` must be described as an experimental +prototype rather than a proven signature-free Starfish variant. Transaction bytes remain outside header RBC. The existing Reed-Solomon dissemination, acknowledgment, reconstruction, and transaction-commitment checks remain responsible for data @@ -112,8 +105,8 @@ strong parents and certified frontier constrain consensus. ## 4. Canonical objects -The milestone-two codec implements the following logical types. Field widths, enum codes, maximum -lengths, and golden bytes are frozen before runtime integration. +The milestone-two codec should implement the following logical types. Field widths, enum codes, +maximum lengths, and golden bytes are frozen by that milestone, before runtime integration. ```rust struct CarrierHeaderV1 { @@ -140,7 +133,8 @@ struct ConsensusVertexV1 { consensus_round: RoundNumber, strong_parents: Vec, delivery_frontier: Vec>, - leader_choice: LeaderChoiceV1, + // None only for the fixed genesis consensus round. + leader_choice: Option, } struct ConsensusVertexReference { @@ -164,30 +158,18 @@ For non-genesis carrier round `r`: - every weak parent has carrier round `r - 1` and a distinct non-local author; - the stake of `{ own_prev } union weak_parents` reaches `Q`; - phase targets have carrier rounds strictly below `r`; and -- weak parents are in strict authority order and duplicate-free; -- acknowledgments commit to the unique normalized sequence described below, and phase batches are - order-significant logs whose exact order is committed; -- a phase batch contains at most one statement for each `(phase, target author, target round)`; and -- strong-parent ordering, frontier indexing, and leader-choice validity are checked only when the - optional consensus vertex is projected, so an ineligible vertex cannot invalidate its carrier. - -The first encoded carrier has round one and names the fixed virtual round-zero carrier reference of -its author as `own_prev`; no round-zero carrier is sent on the wire. Consensus genesis is likewise -virtual. Every embedded consensus vertex has a positive consensus round and an explicit leader -choice. These conventions avoid making genesis a second, partially authenticated wire format. +- every vector is canonically ordered and duplicate-free. Weak references are syntax and pacing declarations, not availability assertions. Their target headers need not be present to authenticate, admit, process, or RBC-deliver the enclosing carrier. -Acknowledgments have one canonical logical order: first the unique maximal suffix shared with -`[own_prev] || weak_parents`, then all remaining acknowledgments in their original relative order. -The content digest commits to this expanded, suffix-first sequence, while the wire codec stores the -shared suffix as an intersection index and retains the order-significant extras. Non-canonical wire -aliases and duplicate acknowledgments are rejected. An honest author creates an acknowledgment -only after the exact target is locally RBC-delivered and its transaction data reconstructs to the -committed root. The acknowledgment becomes usable as data-availability evidence only after its -enclosing carrier is also locally RBC-delivered; an optimistically admitted Byzantine carrier -cannot create inconsistent availability facts at different validators. +Acknowledgments retain Starfish's logical order and compression on the wire. The content digest +commits to the expanded logical vector, while the stored compression must be canonical. An honest +author creates an acknowledgment only after the exact target is locally RBC-delivered and its +transaction data reconstructs to the committed root. The acknowledgment becomes usable as +data-availability evidence only after its enclosing carrier is also locally RBC-delivered; an +optimistically admitted Byzantine carrier cannot create inconsistent availability facts at +different validators. `delivery_frontier` has exactly one indexed entry per committee authority. `None` denotes that authority's fixed genesis/empty prefix. A `Some(reference)` entry must name the same authority as @@ -211,42 +193,10 @@ phase batch and optional consensus vertex. It excludes: - receipt peer, arrival time, and local admission state; and - recovery or transport metadata. -The byte grammar uses a one-byte format version, fixed field markers, big-endian fixed-width -integers, and explicit vector lengths. It does not add a `starfish:block-ref:v2` string to the block -identity. The format byte and unambiguous grammar distinguish this carrier layout; changing the -layout requires a new version and new golden vectors. The canonical identity codec is handwritten; -serde or bincode framing is never hashed. - -Milestone two freezes the version-one identity grammar as follows. `Ref` is -`author:u16 || carrier_round:u32 || digest:[u8;32]`; every integer is big-endian and every vector -count is `u16`. - -```text -00 01 -01 author:u16 -02 carrier_round:u32 -03 own_prev:Ref -04 weak_count:u16 weak:Ref[] -05 transactions_commitment:[u8;32] -06 acknowledgment_count:u16 expanded_acknowledgments:Ref[] -07 phase_count:u16 (phase:u8 target:Ref)[] // ECHO=0, READY=1 -08 consensus_present:u8 [ConsensusVertexV1] -09 creation_time_ns:u64 -``` - -The optional consensus encoding uses markers `01` through `04` for consensus round, strong -parents, delivery frontier, and leader choice. A strong reference is `Ref || consensus_round:u32`; -frontier entries use `0=None` and `1=Some(Ref)`; leader choices use `1=Vote` and `2=NoVote` (`0` is -reserved for virtual genesis and is rejected on the wire). The canonical transport codec replaces -the expanded acknowledgment field with `intersection_start:u16 || extra_count:u16 || extras`, where -the intersection is the unique maximal suffix of `[own_prev] || weak_parents`. Decoding expands and -recompresses this field and rejects aliases. To keep the two byte grammars self-describing, this -compressed transport form starts with `00 81`; only expanded identity content starts with `00 01`. - -Version one caps canonical carrier content at 4 MiB, weak and strong parents at the committee size, -the frontier at exactly the committee size when projected, and encoded phase batches at -`min(4n, 2048)`. The `4n` bound gives two times the expected `2n` steady-state phase arrival rate; -the scheduler still needs the fair-prefix and active-window rules described in Section 8.3. +The byte grammar uses fixed field markers, fixed-width integers, and explicit vector lengths. It +does not add a `starfish:block-ref:v2` string to the block identity. A format-version field and the +unambiguous grammar distinguish this carrier layout; changing the layout requires a new version and +new golden vectors. Consensus vertices are referenced by their exact enclosing `BlockReference` plus their declared `consensus_round`. Because there is at most one consensus vertex per carrier, that pair identifies @@ -465,18 +415,12 @@ authenticated holders eventually obtains the value after GST. ### 8.3 Batching and fairness -Phase batches are bounded. The encoded order is preserved and processed as an authenticated log; -two different orders intentionally identify different carriers. A deterministic fair queue must -prevent Byzantine traffic for one slot +Phase batches are bounded. A deterministic fair queue must prevent Byzantine traffic for one slot from starving honest ECHO/READY actions for other slots. In steady state, one authority can owe one ECHO and one READY for each of `n` previous-round carriers, so `2n` is the expected arrival rate and -not a safe capacity. The executable model retains an unbounded pending FIFO and drains the first -`4n` statements eligible for the carrier being built (capped by the version-one codec limit of -2,048 statements). A temporarily ineligible future-round statement remains in its stable queue -position but does not block older eligible work behind it. This exercises backlog, runahead, and -batching without pretending to solve adversarial fairness. A bounded runtime must use a fair -per-slot scheduler, reserve strictly more than `2n` statements per carrier, and enforce an -active-slot window so delayed work drains instead of remaining at permanent saturation. +not a safe capacity. The executable model initially uses an unbounded fair queue. A bounded runtime +must reserve strictly more than `2n` statements per carrier, plus an active-slot window, so delayed +work drains instead of remaining at permanent saturation. ## 9. Certified consensus vertices @@ -686,20 +630,11 @@ vector dissemination is a later optimization and requires redundant routes or di An authoritative implementation must persist proof-critical choices before exposing effects: -1. journal typed authenticated inbound provenance, exact bytes, and its local ingress sequence; -2. before fixing a local slot, construct and persist the typed candidate plus its exact canonical - carrier bytes and reference; -3. persist local ECHO, READY, explicit leader-choice, delivery, carrier-slot, and consensus-slot - locks that match that retained candidate (recovered content is likewise retained before READY); -4. persist the exact authentication sidecar and an outbound-exposure marker, and only then send the - carrier; and -5. after restart, replay the journal in recorded order and retransmit the identical carrier and - sidecar. - -Persisting a bare local reference before its canonical carrier bytes is not sufficient: a crash in -that gap would leave the slot fixed without the data needed to reconstruct the exact carrier. The -write-ahead model therefore makes content retention precede slot fixation and prevents exposure -until every lock encoded by that carrier is durable. +1. journal authenticated inbound provenance and its local ingress sequence; +2. persist local ECHO, READY, explicit no-vote, delivery, carrier-slot, and consensus-slot locks; +3. construct and persist the exact outbound carrier bytes, reference, and authentication sidecar; +4. only then send the carrier; and +5. after restart, replay the journal in recorded order and retransmit the identical carrier. Every persisted slot, candidate, lifecycle predicate, journal entry, and outbound-carrier key is namespaced by both `protocol_instance` and `committee_id`; storage from another run or committee @@ -791,8 +726,7 @@ minimum it must cover: - equal committed anchors producing byte-identical output deltas; - delayed data availability followed by eventual prefix inclusion; - crash points before and after each persisted lock and outbound-carrier write; and -- once milestone three supplies the non-authoritative runtime path, shadow replay matching the - current direct RBC kernel's delivered references. +- shadow replay matching the current direct RBC kernel's delivered references. Property tests should mutate every canonical field and verify carrier-reference binding, while golden tests freeze the version-one encoding and flat vector length. @@ -826,13 +760,11 @@ Every milestone is committed separately. 1. **Protocol specification (this document):** lock the two clocks, lifecycle, full-vector sidecar, embedded Bracha transitions, certified prefix/frontier, commit/skip boundary, proof obligations, and experiment plan. No protocol code or CLI selector is added. -2. **Canonical codec and executable model (implemented):** isolated carrier, phase, consensus, - frontier, and sidecar types; golden encodings; pure carrier/RBC, projection/decision, and durable - journal models; and deterministic adversarial simulations. No network or existing consensus path - changes. +2. **Canonical codec and executable model:** add isolated carrier, phase, consensus, frontier, and + sidecar types; golden encodings; a pure in-memory state machine; and deterministic adversarial + simulations. No network or existing consensus path changes. 3. **Persisted shadow carrier path:** build and store carriers alongside the current direct - `starfish-rbc` service, cache the validated committee/domain identity rather than re-hashing all - public keys per carrier, journal ingress and local locks, and compare embedded versus direct RBC + `starfish-rbc` service, journal ingress and local locks, and compare embedded versus direct RBC delivery. Direct RBC remains authoritative; shadow results never affect proposals or commits. 4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses @@ -853,8 +785,8 @@ Every milestone is committed separately. The following values are not safe to guess in the documentation milestone and must be resolved by the executable model or measured prototype: -- production maximum future-carrier buffer and payload runahead (the executable model deliberately - uses admission lookahead `2` and hard buffer lookahead `4` only as test parameters); +- exact canonical field widths and maximum phase-batch size; +- maximum future-carrier buffer and payload runahead; - the control-heartbeat rate under low load and backpressure; - a safe state-retirement, garbage-collection, and late-catch-up watermark; - whether all supported storage backends are required before authoritative mode; From 06cfb6d39a125bdeb9bc39a4cdf44bf99c6d629b Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:46:23 +0200 Subject: [PATCH 61/62] Revert "Document Starfish-RBC-DAG protocol design" This reverts commit 2bb7ae8233b4e328971f63e023f23225e6f2de75. --- README.md | 5 - docs/starfish-rbc-dag-protocol.md | 798 ------------------------------ 2 files changed, 803 deletions(-) delete mode 100644 docs/starfish-rbc-dag-protocol.md diff --git a/README.md b/README.md index 7ad5fe03..85d19d07 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,6 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). -**Starfish-RBC-DAG** is a design-only follow-up that pipelines all-carrier RBC through an optimistic -carrier DAG while keeping certified Starfish consensus and ordering in a separate logical -projection. Its provisional CLI name is `starfish-rbc-dag`, but that selector is not implemented -yet. The full design and proof obligations are documented in the -[protocol design](docs/starfish-rbc-dag-protocol.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/docs/starfish-rbc-dag-protocol.md b/docs/starfish-rbc-dag-protocol.md deleted file mode 100644 index 970dc964..00000000 --- a/docs/starfish-rbc-dag-protocol.md +++ /dev/null @@ -1,798 +0,0 @@ -# Starfish-RBC-DAG protocol design - -> **Frozen comparison baseline.** This document specifies the experimental -> two-plane carrier implementation. New development targets the one-block, -> one-identity design in `starfish-rbc-single-dag-v3.md`; V1/V2 carrier and -> projection formats remain available only for compatibility and matched -> performance comparisons. - -Status: standalone MAC-vector RBC-DAG prototype with authoritative optimistic delivery and -committed-frontier output; the end-to-end proof, proof-safe retirement, checkpoint transfer, and -full validator recovery remain incomplete - -The provisional CLI name for this protocol is `starfish-rbc-dag`. It is a new protocol, not a -transport option or a version-two alias for `starfish-rbc`. - -The implemented [`starfish-rbc`](starfish-rbc-protocol.md) prototype remains the conservative -baseline: it sends Bracha INIT/ECHO/READY as direct network messages, advances Starfish only through -RBC-delivered dependency-closed headers, and sends one initial MAC tag to each recipient. This -document instead specifies an optimistic carrier DAG that embeds the reliable-broadcast transcript, -uses a complete committee-sized MAC vector on every MAC-authenticated carrier, and separates fast -carrier pacing from certified consensus ordering. - -The two designs may share canonicalization, cryptography, storage, and benchmark code, but they are -not wire-compatible and do not have the same proof obligations. Nothing in this document changes -the behavior of `starfish-rbc` or any existing protocol. - -## 1. Objective and non-claims - -The objective is to recover the pipelining of an uncertified Starfish DAG without allowing -optimistically received Byzantine blocks to affect safety: - -- a fast physical carrier DAG advances on a quorum of locally authenticated carriers; -- every application carrier is also the value of a Bracha reliable-broadcast instance; -- later carriers batch ECHO and READY statements for earlier carriers; -- only RBC-delivered, data-available information enters the logical consensus projection; and -- committed consensus frontiers eventually order every honest on-prefix application carrier. - -The initial research mode sends the complete ordered MAC vector with every carrier. The vector is -an authentication sidecar and is not part of the carrier digest. Ed25519, ML-DSA-44, and ML-DSA-65 -remain selectable outer-authentication baselines; changing that selector does not change the -embedded RBC or consensus rules. - -This is a proposed composition. The reliable-broadcast thresholds are standard, and the Starfish -commit rules already exist, but their composition through two clocks, two logical projections, and -frontier-based payload ordering still requires an executable model, adversarial tests, and a proof. -Until those obligations are discharged, `starfish-rbc-dag` must be described as an experimental -prototype rather than a proven signature-free Starfish variant. - -Transaction bytes remain outside header RBC. The existing Reed-Solomon dissemination, -acknowledgment, reconstruction, and transaction-commitment checks remain responsible for data -availability. - -## 2. Model and notation - -Version one assumes: - -- one static, ordered, stake-weighted committee for a run; -- Byzantine stake strictly below one third of total stake; -- pairwise symmetric keys for every ordered validator pair; -- reliable authenticated point-to-point communication after GST; -- a fresh nonzero protocol-instance identifier shared through genesis configuration; -- no committee reconfiguration; and -- no state retirement until a safe recovery watermark is proved. - -For total committee stake `W`, use the repository's integer thresholds: - -```text -Q = floor(2W / 3) + 1 -V = floor(W / 3) + 1 -``` - -For equal stake and `n = 3f + 1`, these are `Q = 2f + 1` and `V = f + 1`. - -Two independent round numbers are used: - -- `carrier_round` belongs to the fast physical DAG and advances from authenticated admission; -- `consensus_round` belongs to the certified logical Starfish projection. - -There is no fixed mapping between them. Carrier rounds may run ahead while a consensus vertex is -waiting for RBC delivery, data availability, a leader decision, or certified strong parents. - -## 3. One physical DAG, two logical projections - -The only network DAG objects are carriers. A carrier contains application-header data, a bounded -batch of RBC control statements, and optionally one consensus vertex. - -The same stored objects have two disjoint interpretations: - -1. **Optimistic carrier projection.** Authenticated carriers and their weak parent references pace - carrier creation and transport RBC statements. This projection is allowed to differ temporarily - between honest validators. -2. **Certified consensus projection.** Eligible consensus vertices, immutable strong parents, and - certified delivery frontiers drive Starfish voting, certification, skip/commit decisions, and - linearization. Honest validators eventually agree on this projection. - -Weak carrier edges never become strong/order edges, even if their targets later deliver. A node -must not construct its own filtered consensus parent set from an optimistic carrier after the fact: -that would make the authenticated content have different consensus meaning at different nodes. - -This separation prevents a Byzantine carrier from poisoning an honest carrier. A quorum-sized weak -parent set can contain up to `f` selectively disseminated or invented Byzantine references. Waiting -for all such references to become RBC-delivered would make the honest child permanently unusable. -Weak edges are therefore permanently nonblocking and nonordering. Only the explicitly encoded -strong parents and certified frontier constrain consensus. - -## 4. Canonical objects - -The milestone-two codec should implement the following logical types. Field widths, enum codes, -maximum lengths, and golden bytes are frozen by that milestone, before runtime integration. - -```rust -struct CarrierHeaderV1 { - author: AuthorityIndex, - carrier_round: RoundNumber, - - // Physical pacing only. `own_prev` is not repeated in `weak_parents`. - own_prev: BlockReference, - weak_parents: Vec, - - transactions_commitment: TransactionsCommitment, - data_acknowledgments: Vec, - phase_batch: Vec, - consensus_vertex: Option, - creation_time_ns: TimestampNs, -} - -enum RbcPhaseStatementV1 { - Echo { target: BlockReference }, - Ready { target: BlockReference }, -} - -struct ConsensusVertexV1 { - consensus_round: RoundNumber, - strong_parents: Vec, - delivery_frontier: Vec>, - // None only for the fixed genesis consensus round. - leader_choice: Option, -} - -struct ConsensusVertexReference { - carrier: BlockReference, - consensus_round: RoundNumber, -} - -enum LeaderChoiceV1 { - Vote { leader: ConsensusVertexReference }, - NoVote { leader_author: AuthorityIndex, leader_round: RoundNumber }, -} -``` - -The author of an embedded phase statement or consensus vertex is the author of its enclosing -carrier. An outer authenticator therefore authenticates the whole batch without a separate tag or -signature per statement. - -For non-genesis carrier round `r`: - -- `own_prev` has the same author and carrier round `r - 1`; -- every weak parent has carrier round `r - 1` and a distinct non-local author; -- the stake of `{ own_prev } union weak_parents` reaches `Q`; -- phase targets have carrier rounds strictly below `r`; and -- every vector is canonically ordered and duplicate-free. - -Weak references are syntax and pacing declarations, not availability assertions. Their target -headers need not be present to authenticate, admit, process, or RBC-deliver the enclosing carrier. - -Acknowledgments retain Starfish's logical order and compression on the wire. The content digest -commits to the expanded logical vector, while the stored compression must be canonical. An honest -author creates an acknowledgment only after the exact target is locally RBC-delivered and its -transaction data reconstructs to the committed root. The acknowledgment becomes usable as -data-availability evidence only after its enclosing carrier is also locally RBC-delivered; an -optimistically admitted Byzantine carrier cannot create inconsistent availability facts at -different validators. - -`delivery_frontier` has exactly one indexed entry per committee authority. `None` denotes that -authority's fixed genesis/empty prefix. A `Some(reference)` entry must name the same authority as -its vector index. The frontier rules in Section 10 are stateful eligibility rules; they are not -part of context-free carrier decoding. - -### 4.1 Carrier identity - -The carrier reference remains content-only: - -```text -BlockReference = (author, carrier_round, BLAKE3(canonical_carrier_content)) -``` - -Canonical carrier content includes every field of `CarrierHeaderV1`, including the ordered RBC -phase batch and optional consensus vertex. It excludes: - -- the MAC vector or public signature; -- protocol instance and committee ID; -- transaction bytes and erasure-coded shards; -- receipt peer, arrival time, and local admission state; and -- recovery or transport metadata. - -The byte grammar uses fixed field markers, fixed-width integers, and explicit vector lengths. It -does not add a `starfish:block-ref:v2` string to the block identity. A format-version field and the -unambiguous grammar distinguish this carrier layout; changing the layout requires a new version and -new golden vectors. - -Consensus vertices are referenced by their exact enclosing `BlockReference` plus their declared -`consensus_round`. Because there is at most one consensus vertex per carrier, that pair identifies -the immutable embedded value without a second mutable lookup key. - -## 5. Authentication sidecar - -Authentication is outside the carrier reference: - -```rust -enum CarrierAuthenticationV1 { - Ed25519(Ed25519Signature), - MlDsa44(MlDsa44Signature), - MlDsa65(MlDsa65Signature), - MacVector(FlatMacVector), -} - -struct FlatMacVector { - // Exactly committee.len() consecutive 32-byte tags in authority-index order. - tags: Vec, -} -``` - -In MAC mode, author `A` computes entry `q` for recipient `Q_q` over a fixed-width statement that -binds at least: - -```text -STARFISH_RBC_DAG_V1 -carrier-authentication kind and scheme -protocol_instance -committee_id -author A -recipient Q_q -carrier_round -canonical carrier content digest -``` - -The full vector accompanies every normally disseminated MAC carrier in version one, including -relayed carriers. A receiver verifies only the entry at its own committee index. It neither verifies -nor vouches for the remaining entries. - -A carrier received directly from its author and the same carrier received through a relay are both -authentication-eligible when the local entry verifies. This is receiver-specific transferable -authentication: it survives a relay for its intended recipient, but it is not a publicly verifiable -signature and provides no non-repudiation. - -The vector is deliberately not an RBC value and has no consistency invariant. A Byzantine author -may attach different vectors to the same content reference, including a vector with a valid tag for -one recipient and garbage for another. Correctness therefore depends only on the local entry and on -the embedded Bracha protocol, never on agreement about the vector bytes. - -Each node persists one exact vector variant with its carrier for restart and relay. The preference -order is locally generated, directly author-received, then first relayed variant with a valid local -entry. Version one does not merge unverified entries from different vectors. Header recovery after -authenticated quorum phase evidence may return canonical content without a vector; that recovery -can unblock RBC delivery but does not create optimistic carrier admission. - -Public-signature modes use the same context-bound carrier statement without a recipient field and -the same embedded RBC/consensus logic. They exist for controlled performance comparison, not as -separate consensus protocols. - -## 6. Local lifecycle and authority matrix - -The implementation must represent these predicates separately: - -```text -Candidate - canonical carrier content and reference are valid - -Authenticated - local author, valid public signature, or valid local MAC-vector entry - -CarrierAdmitted - Candidate && Authenticated && accepted by the carrier admission window - -Delivered - local Bracha instance reached Q READY and pinned matching canonical content - -PrefixClosed - Delivered && DataAvailable && exact own_prev prefix is closed - -VertexProjected (orthogonal to the carrier lifecycle) - this carrier's optional consensus vertex is eligible in the certified projection - -Included - a committed Starfish anchor frontier names this carrier in its deterministic delta - -Ordered - the complete included delta is available and the carrier has been deterministically output -``` - -`Candidate` alone permits bounded staging and digest-based recovery. `CarrierAdmitted` permits -immediate phase-batch processing and fast-pacemaker counting. `Delivered` permits phase replay even -at a node whose author MAC entry was poisoned. `PrefixClosed` permits frontier inclusion. -`VertexProjected` alone permits the optional vertex to supply Starfish -vote/certifier/leader evidence. It is not a later state of every carrier: a carrier with no eligible -optional vertex may still become prefix-closed, included by another anchor's frontier, and ordered. - -The existing generic `dirty` bit is not a synonym for `CarrierAdmitted`: current RBC code may stage -content after invalid initial authentication or during recovery. Reusing that bit would let an -unauthenticated candidate advance the fast clock. - -| Consumer | Required local authority | -|---|---| -| Header retention/recovery | `Candidate` | -| Process embedded RBC statements | `CarrierAdmitted`, or `Delivered` for replay | -| Fast carrier clock | `CarrierAdmitted` | -| Transaction/shard synchronization | `Candidate` | -| Count a data-availability acknowledgment | `Delivered` enclosing carrier | -| Delivery frontier | `PrefixClosed` target carrier | -| Starfish QC, skip, and anchor commit | `VertexProjected` consensus vertex | -| Application payload output | `Included` delta, with every member delivered/data-available | - -The initial implementation should keep fast-pacemaker counting exactly at `CarrierAdmitted`; using -`Delivered` as an additional stronger pacing input can be added only if the executable model shows -that it cannot change the sequential slot accounting. - -## 7. Fast carrier pacemaker - -The carrier clock is sequential and does not use the current threshold-clock helper's ability to -jump to a far-future round after one message. - -For each carrier round, a validator records at most one admitted reference per author. Byzantine -equivocations may make different honest validators record different references for a Byzantine -author, but each author contributes stake once. A validator advances from carrier round `r` to -`r + 1` only after: - -1. its own carrier at round `r` has been fixed and persisted; and -2. it has admitted distinct-author carrier stake `Q` at round `r`. - -Future carriers are bounded and buffered; they do not skip missing local rounds. The next local -carrier records the selected quorum as `{ own_prev } union weak_parents`. A missing weak-parent body -never blocks the next carrier or any later consensus action. - -This clock replaces the current `starfish-rbc` rule that requires a quorum of RBC-clean previous -round headers before proposal. It does not make admitted carriers consensus votes. Leader/vote/skip -waiting conditions move to the independent consensus projection and cannot block the creation of a -carrier needed to transport ECHO or READY. - -Honest validators emit empty control heartbeats when they have no application transactions. -Without heartbeats, low load can stop the ECHO/READY waves and violate RBC liveness. Every carrier, -including an empty heartbeat, is itself an RBC value and has an authenticator sidecar. - -The first prototype retains the current run's carrier/RBC state and rate-limits carrier creation. -A production design needs a proved runahead and backpressure rule. A hard carrier/consensus skew cap -must not suppress control heartbeats, because those heartbeats may be exactly what allows the -certified frontier to catch up. - -## 8. Embedded all-carrier reliable broadcast - -There is one RBC slot for every physical carrier: - -```text -RbcSlot = (protocol_instance, committee_id, carrier_author, carrier_round) -RbcValue = BlockReference -``` - -Authenticating the carrier is INIT for that value. The local author records its own ECHO when it -atomically fixes and persists the carrier. An honest non-author that first admits a value queues one -ECHO for that slot. - -INIT is not silently counted as the author's ECHO at remote validators. The author's recorded local -ECHO is queued into a later carrier like every other phase action; it counts locally immediately and -remotely only after the enclosing carrier is admitted or delivered. This preserves the standard -quorum accounting without excluding the broadcaster's stake. - -Local phase actions are inserted into the next possible carrier's `phase_batch`. The enclosing -carrier's authentication makes its author the phase sender. For each target slot: - -- an authority emits at most one ECHO; -- an authority emits at most one READY; -- ECHO and READY choices may differ; -- `Q` ECHO stake creates a READY obligation; -- `V` READY stake creates a READY obligation; and -- `Q` READY stake plus pinned matching content locally delivers the exact carrier. - -A READY obligation is not yet a local READY. If the target content is absent, the validator first -recovers it from authenticated ECHO/READY authors, validates the exact reference, and durably pins -it. Only then does it persist the slot-global READY lock, count its local READY, and enqueue the -statement. Consequently every honest READY author is a real content holder. A quorum trigger remains -latched while recovery is pending. - -Local ECHO and READY count toward their thresholds before network dissemination. Evidence is -tracked per candidate, while local send and delivery locks are slot-global. Each remote authority -contributes stake at most once per phase and target slot; exact replay is idempotent and a later -equivocation is ignored before allocating another candidate. - -### 8.1 Processing rule - -After canonical validation and outer authentication, a receiver processes the phase batch in its -canonical encoded order immediately. It must not wait for the enclosing carrier itself to be RBC -delivered, data-available, dependency-closed, or projected. Waiting would create a recursion: a -carrier's controls are required to deliver earlier carriers, while those earlier deliveries may be -required to create the next consensus vertex. - -If the local carrier authenticator is missing or invalid, its phase batch is not processed on -candidate receipt. If that exact outer carrier later becomes locally RBC-delivered, the stored batch -may be replayed under the local delivery capability. Thus poisoned vector entries delay optimistic -admission but cannot permanently suppress controls selected by RBC. - -Phase targets are strictly older carrier rounds, making a single carrier's replay acyclic. Local -arrival order between different authenticated carriers is still observable and can affect which -Byzantine equivocation encounters a slot-global guard first. Recovery must replay the persisted -ingress journal, never reconstruct choices by sorting carriers after a restart. - -### 8.2 Header recovery - -An honest ECHO or READY author must retain the target carrier content. A validator that observes a -threshold before receiving the target requests it from several recorded phase authors. Recovery -content is accepted only when canonical validation recomputes the requested reference. - -Recovery request/response remains an out-of-band data-transfer optimization in the first -prototype. It is not quorum testimony and does not change the on-DAG phase transcript. A `Q` ECHO -set contains honest holders, and a `V` READY set contains at least one honest holder, so retrying -authenticated holders eventually obtains the value after GST. - -### 8.3 Batching and fairness - -Phase batches are bounded. A deterministic fair queue must prevent Byzantine traffic for one slot -from starving honest ECHO/READY actions for other slots. In steady state, one authority can owe one -ECHO and one READY for each of `n` previous-round carriers, so `2n` is the expected arrival rate and -not a safe capacity. The executable model initially uses an unbounded fair queue. A bounded runtime -must reserve strictly more than `2n` statements per carrier, plus an active-slot window, so delayed -work drains instead of remaining at permanent saturation. - -## 9. Certified consensus vertices - -A carrier contains zero or one `ConsensusVertexV1`. The carrier remains valid and pace-eligible if -the optional vertex is malformed relative to local certified state; only the optional vertex is -excluded from the consensus projection. - -A consensus vertex authored by `A` at consensus round `c > 0` is eligible only when: - -1. its enclosing carrier is locally RBC-delivered; -2. its enclosing carrier's transaction data is available and it closes `A`'s carrier prefix as - defined in Section 10; -3. its strong parents name distinct-author eligible consensus vertices at exactly `c - 1` whose - stake reaches `Q`; -4. the strong-parent set includes `A`'s preceding consensus vertex for non-genesis `c`; -5. its delivery frontier is closed and dominates every strong parent's effective frontier; and -6. its leader choice is valid for the deterministic leader role at `c - 1`. - -For a vote, the exact leader must be an eligible strong parent at `c - 1`. No-vote validation is -objective and structural: it names the correct leader slot and no value from that slot appears in -the immutable strong-parent set. Remote validators do not attempt to verify that the author's local -timeout expired. If the strong-parent frontiers contain incomparable components, their join is -undefined and the optional vertex is ineligible; honest construction waits for a compatible quorum -rather than importing the fork into consensus. - -Strong parents and voted leaders decrease strictly in `consensus_round`, which makes the consensus -projection acyclic. Their enclosing `carrier_round` may be numerically higher than the child's -because the clocks are independent and honest carrier authors can be skewed. Strong edges are never -interpreted as physical weak/self edges or application-order dependencies. - -Consensus references and strong parents are immutable authenticated content. Missing or ineligible -strong parents block only this optional vertex. They never block the enclosing carrier, its phase -batch, the fast clock, or later honest RBC progress. - -The consensus pacemaker preserves Starfish's separate advance and creation conditions, evaluated -only over eligible consensus vertices: - -- **A1:** advance from `c - 1` to `c` after eligible distinct-author stake `Q` at `c - 1`; -- **A2:** do not advance until the local consensus vertex at `c - 1` has been fixed; -- **C1:** create at `c` after the eligible leader at `c - 1` is present and the eligible projection - contains either `Q` votes for an exact leader value or a valid explicit direct-skip pattern for - the leader slot at `c - 2`; -- **C2:** create after the consensus leader timeout; or -- **C3:** catch up and create after observing eligible distinct-author stake `Q` already at `c`. - -The strong-parent set chosen under C1 must itself contain the immutable L2 witness: the exact `Q` -voter vertices for a certificate, or the union of explicit negative-choice witnesses required by -the direct-skip evaluator. It must also contain the eligible leader at `c - 1`. Strong-parent sets -therefore contain between `Q` and `n` distinct authors. Merely observing the witness elsewhere in -the local projection is insufficient, because later certifiers must inherit it through the new -vertex's strong history. - -If the eligible leader at `c - 1` is present when the local vertex at `c` is fixed, the vertex must -include that exact leader as a strong parent and record `Vote`. It may record `NoVote` only when it -is created through the timeout/catch-up path without that leader in its immutable strong-parent -set. This timeout/catch-up restriction is an honest-author creation rule; Byzantine authors may -emit structurally valid no-votes arbitrarily. These conditions prevent an adversarially scheduled -quorum that excludes each just-late leader from turning every consensus round into a skip. - -The next local consensus vertex is embedded whenever the carrier scheduler next runs after its -creation condition becomes true. There is no requirement that its carrier round equal `c`, `c + 1`, -or any other fixed offset. Fast carrier production continues while C1/C2/C3 are unsatisfied. - -A Byzantine author may embed conflicting consensus values for the same `(author, consensus_round)` -in different carrier rounds. All structurally eligible conflicts remain visible as equivocation; -there is no local first-arrival or anchor-time pruning rule. An honest author creates at most one -value in its local slot, every strong-parent or evidence set contains at most one value per author, -and stake aggregation counts each author once. Votes and committed leaders name exact references, -so the existing equivocation-aware Starfish safety argument—not an invented canonical -choice—must resolve Byzantine conflicts. - -## 10. Closed delivery prefixes and frontiers - -RBC delivery alone is not a compact availability proof for a Byzantine author's later carrier. A -Byzantine author may deliver round `r` with an `own_prev` that names an unavailable fork at -`r - 1`. Therefore a frontier component is a contiguous exact prefix, not simply the highest -delivered round. - -For authority `A`, begin at its fixed genesis/empty prefix. A carrier `(A, r, R)` extends the local -closed prefix only when: - -- `R` is locally RBC-delivered; -- its transaction data satisfies the existing Starfish availability predicate; -- `r` is exactly one more than the current prefix round; and -- `R.own_prev` equals the exact current prefix tip. - -Later delivered carriers above a gap remain stored but do not advance the prefix. A Byzantine -off-prefix fork may be discarded from application ordering without affecting honest-carrier -liveness. - -The join of strong-parent effective frontiers is computed componentwise. A child frontier dominates -that join only when each entry is the same exact tip or an exact self-chain extension of it; -comparing round numbers alone is insufficient. Including each parent's enclosing carrier prevents a -child from omitting a strong parent from its eventual frontier closure. Honest authors advertise the -newest locally closed tip for every authority, subject to that dominance rule. This monotonic rule -ensures that committed frontiers never regress or switch Byzantine forks. - -The containing carrier cannot name itself in its encoded frontier. For an eligible consensus -vertex, its declared author component must equal its carrier's `own_prev` prefix tip. Once the -enclosing carrier is delivered and data-available, its **effective frontier** replaces that one -component with the enclosing carrier. This makes a committed anchor's own application payload -eligible without waiting for a later anchor while preserving exact prefix continuity. - -The liveness target is deliberately precise: - -> Every honest carrier that RBC-delivers and becomes data-available eventually appears in a -> committed effective-frontier delta. - -No guarantee is made for a malformed or permanently off-prefix Byzantine carrier. Guaranteeing all -RBC-delivered Byzantine forks would require an antichain or sparse exception structure rather than -one compact prefix tip per authority. - -## 11. Starfish certification, commit, and skip - -Starfish's logical leader schedule and commit rules run over eligible consensus vertices only. -Carrier admission, weak parents, phase targets, candidate headers, and merely delivered carriers -cannot act as voters, certifiers, leaders, non-votes, or reachability evidence. - -For a scheduled leader slot at consensus round `c`, every eligible voter publishes one immutable -slot choice. `Vote(L)` is positive evidence only for the exact leader value `L` and explicit -negative evidence for every conflicting value in that leader slot. `NoVote(slot)` is negative -evidence for every value in the slot. Thus a late Byzantine equivocation cannot turn an earlier -omission into a new choice. - -The existing Starfish patterns are then evaluated from these explicit choices: - -- an eligible vertex at `c + 1` explicitly records `Vote(L)` or `NoVote(leader_slot)`; -- `Q` distinct-author votes certify `L`; -- an eligible vertex at `c + 2` whose certified history contains `Q` such votes is a certifier; -- `Q` distinct certifier authors provide the direct-commit condition; and -- a per-candidate quorum of explicit negative choices provides the direct-skip pattern. - -If the leader produces no value, `Q` immutable `NoVote(slot)` choices are a self-contained direct -skip witness. If a Byzantine leader equivocates, `Vote(L)` is negative evidence for every other -candidate, and the current Starfish per-candidate evaluator decides whether the collected explicit -choices form a direct-skip pattern; otherwise the slot remains for indirect decision. - -Indirect commit/skip follows the existing Starfish rule over this same eligible strong-parent -projection. An omission from a phase batch, weak parent list, missing carrier, or locally filtered -view is never a no-vote. `NoVote` is explicit, authenticated, immutable, and slot-locked. -An honest validator persists its leader-choice lock before exposing the carrier that contains it; -it cannot emit `NoVote` and later vote for a late leader in the same logical voting slot. - -Skipping a Byzantine leader role discards only that optional consensus value. It does not discard -the enclosing application carrier. If that carrier later becomes part of a closed prefix, a later -committed frontier orders its payload. - -Every consensus consumer in the current Starfish committer must be audited for the new type -boundary: voter caches, leader support, potential certificates, direct/indirect decisions, -reachability, and the linearizer must reject non-projected carrier facts. Data-availability -acknowledgments are the deliberate exception: they become usable when their enclosing carrier is -RBC-delivered, which breaks a projection/availability circularity while still excluding merely -optimistic evidence. - -## 12. Frontier-delta linearization - -Let `F_k` be the effective frontier carried by committed anchor `A_k`, and let `Closure(F_k)` be the -union of the exact per-author self-chain prefixes named by `F_k`. Maintain: - -```text -C_0 = fixed genesis carriers -C_k = C_(k-1) union Closure(F_k) -Delta = C_k \ C_(k-1) -``` - -Before outputting `Delta`, a validator waits until every exact member is locally RBC-delivered and -data-available. RBC totality and erasure-coded recovery supply missing content for honest committed -frontiers. - -All validators deterministically order the same delta by -`(carrier_round, author, content_digest)`. Because a closed author prefix advances by exactly one -carrier round, this key already preserves mandatory `own_prev` order. - -Weak parents, strong consensus edges, optional-vertex projection time, ECHO/READY target references, -recovery provenance, and MAC-vector variants never constrain application payload ordering. Strong -edges order consensus decisions and dominate frontiers, but a late-projecting optional vertex must -not retroactively add an edge between payloads already output. This fixed ordering also ensures that -a dangling Byzantine weak edge cannot reintroduce the liveness failure that the two-projection -design removes. - -## 13. Expected optimistic schedule - -In an all-honest synchronous interval, batching can realize this conceptual schedule: - -```text -t = 0 carrier k contains a new application header (RBC INIT) -t = delta carrier k+1 contains ECHOs for k -t = 2delta carrier k+2 contains READYs for k -t = 3delta carrier k is RBC-delivered; a later carrier may project new consensus work -``` - -The embedded design does not make Bracha RBC require fewer communication delays than the direct -baseline. Its performance hypothesis is that carrier batching reduces frames, scheduling work, and -duplicated control metadata while the fast carrier clock overlaps certification with dissemination. - -Implementation ordering is latency-critical. On carrier ingress, authenticate, apply its phase -batch, execute newly enabled delivery/prefix/projection transitions, and only then decide what the -next local carrier should contain. Constructing the next carrier first would accidentally add a -full carrier round to every RBC wave. - -The complete vector costs `32n` bytes in every MAC carrier copy. Under all-to-all dissemination this -can erase much of the batching gain. The first benchmark is therefore a whole-protocol result, not -evidence that full-vector all-to-all transport is asymptotically better. Tree or bounded-fanout -vector dissemination is a later optimization and requires redundant routes or direct fallback. - -## 14. Persistence, recovery, and boundedness - -An authoritative implementation must persist proof-critical choices before exposing effects: - -1. journal authenticated inbound provenance and its local ingress sequence; -2. persist local ECHO, READY, explicit no-vote, delivery, carrier-slot, and consensus-slot locks; -3. construct and persist the exact outbound carrier bytes, reference, and authentication sidecar; -4. only then send the carrier; and -5. after restart, replay the journal in recorded order and retransmit the identical carrier. - -Every persisted slot, candidate, lifecycle predicate, journal entry, and outbound-carrier key is -namespaced by both `protocol_instance` and `committee_id`; storage from another run or committee -cannot satisfy a local lock or quorum. - -Hash-sorting recovered carriers is not a valid reconstruction rule. Byzantine equivocation can make -arrival order determine which value a local slot-global guard selects, and a different restart order -could make one honest authority appear to send conflicting phases. - -The initial model and shadow prototype retain all proof-critical carrier, phase, prefix, and -consensus state for the run. Before garbage collection is enabled, the design needs a common -retirement watermark that preserves: - -- pending Bracha totality and header recovery; -- exact self-prefix expansion from the last committed frontier; -- committed-anchor reconstruction for a late validator; and -- deterministic replay of local locks. - -Resource bounds still required before authoritative deployment include a future carrier window, -per-peer candidate caps, a fair phase backlog, a rate-limited control heartbeat, a bounded payload -runahead policy, and disk-backed recovery. Resource exhaustion is excluded from the initial proof -model but must be measured in the prototype. - -## 15. Safety obligations - -The design is not complete until at least the following claims are proved or falsified by a model: - -1. **Receiver-authentication integrity.** An honest receiver admits a carrier attributed to an - honest author only if that author created the public proof or the receiver's MAC entry. A MAC is - not public non-repudiation, and a Byzantine endpoint knows its own pairwise key. -2. **RBC agreement and integrity.** Slot-global ECHO/READY locks, quorum intersection, and exact - value binding prevent two conflicting carrier values from being delivered by honest validators. -3. **RBC totality.** If one honest validator delivers a value, heartbeats, READY amplification, and - holder recovery cause every honest validator eventually to deliver the same value. -4. **Optimistic isolation.** Carrier admission can change only fast pacing and RBC processing; it - cannot alter a QC, leader decision, skip, commit, acknowledgment certificate, or output order. -5. **Weak-edge non-poisoning.** A missing or equivocating weak parent cannot block delivery, - projection of unrelated honest vertices, or application ordering. -6. **Consensus-slot uniqueness.** Honest validators create/vote once per - `(author, consensus_round)`, and Byzantine conflicts cannot both acquire honest quorum support. -7. **Prefix comparability.** Every accepted frontier component is an exact extension of its strong - ancestors and of every earlier committed component. -8. **Projection safety.** Erasing weak edges and optional consensus metadata that is not - `VertexProjected` leaves a valid execution of the Starfish commit/skip rules over immutable - strong edges; it does not erase otherwise orderable carrier payloads. -9. **Deterministic ordering.** Equal committed anchors imply equal frontier closures, deltas, and - transaction order at all honest validators. -10. **Data availability.** No carrier enters an output delta until its committed transaction root - can be reconstructed and verified. - -## 16. Liveness obligations - -Under partial synchrony and fair processing, the design must establish: - -1. `Q` honest authors continually create authenticated carriers after GST, so the sequential fast - clock advances without Byzantine participation. -2. Empty heartbeat carriers drain every honest ECHO/READY backlog even when application load is - zero. -3. Every honest carrier is RBC-delivered at every honest validator. -4. Existing Starfish data availability eventually closes every honest author's exact carrier - prefix. -5. Honest consensus vertices with quorum strong parents continue to appear despite arbitrary - Byzantine weak parents, malformed optional vertices, and carrier/consensus round skew. -6. The projected Starfish pacemaker eventually commits infinitely many honest anchors. -7. Honest frontier construction is fair: every newly closed honest carrier prefix is eventually - included in a committed frontier. -8. Waiting for a committed delta cannot block forever because every named exact carrier is already - RBC-delivered and data-available by frontier eligibility. - -The guaranteed payload-liveness statement covers every honest on-prefix carrier. Selectively -disseminated, malformed, or off-prefix Byzantine carriers may be ignored. - -## 17. Required executable tests - -Milestone two begins with an isolated deterministic model, not production network wiring. At -minimum it must cover: - -- `n = 4, f = 1` and `n = 7, f = 2` all-honest progress; -- split Byzantine INIT values and receiver-selective poisoned vector entries; -- valid relayed local MAC entries and invalid vector variants; -- ECHO/READY equivocation, replay, reordering, and evidence-before-header recovery; -- zero application load with heartbeat-only RBC completion; -- future carriers that cannot jump the local sequential clock; -- `f` permanently missing weak parents without blocking honest carrier or consensus progress; -- a delivered Byzantine carrier above an unavailable self-chain gap; -- conflicting Byzantine consensus vertices in one logical slot; -- explicit vote/no-vote conflicts and direct plus indirect commit/skip; -- frontier fork, regression, and strong-parent dominance rejection; -- equal committed anchors producing byte-identical output deltas; -- delayed data availability followed by eventual prefix inclusion; -- crash points before and after each persisted lock and outbound-carrier write; and -- shadow replay matching the current direct RBC kernel's delivered references. - -Property tests should mutate every canonical field and verify carrier-reference binding, while -golden tests freeze the version-one encoding and flat vector length. - -## 18. Complexity and benchmark plan - -The first fair benchmark matrix includes: - -- plain Starfish with Ed25519 and each ML-DSA choice; -- the unsafe `starfish-mac` dissemination lower bound; -- implemented direct `starfish-rbc` with the same authentication choices; -- `starfish-rbc-dag` in MAC-vector and signature modes; and -- Sailfish++ as a certified signature-free comparison. - -Hold committee, load, transaction size, topology, latency injection, dissemination fanout, duration, -timeouts, and build constant. Report carrier/INIT, vector, ECHO, READY, recovery, transaction/shard, -and synchronization bytes separately. Also report authentication CPU, fast-admission-to-delivery -latency, carrier/consensus round skew, prefix lag, commit latency, throughput, and peak retained -state. - -Batching can reduce the number of separately scheduled RBC control messages, but it does not remove -their logical quorum evidence. Full-vector all-to-all transport sends `n` tags in each of `n - 1` -copies per carrier, so it is not expected to improve author egress until a tree or bounded-fanout -transport is added. Shadow mode also sends both direct and embedded transcripts and is a correctness -instrument, not a performance result. - -## 19. Contained implementation milestones - -Every milestone is committed separately. - -1. **Protocol specification (this document):** lock the two clocks, lifecycle, full-vector sidecar, - embedded Bracha transitions, certified prefix/frontier, commit/skip boundary, proof obligations, - and experiment plan. No protocol code or CLI selector is added. -2. **Canonical codec and executable model:** add isolated carrier, phase, consensus, frontier, and - sidecar types; golden encodings; a pure in-memory state machine; and deterministic adversarial - simulations. No network or existing consensus path changes. -3. **Persisted shadow carrier path:** build and store carriers alongside the current direct - `starfish-rbc` service, journal ingress and local locks, and compare embedded versus direct RBC - delivery. Direct RBC remains authoritative; shadow results never affect proposals or commits. -4. **Optimistic carrier clock:** add the distinct authenticated-admission latch, sequential quorum - clock, heartbeats, bounded future buffer, and carrier synchronization while consensus still uses - the current baseline. -5. **Authoritative embedded RBC:** remove direct ECHO/READY authority only after shadow tests show - identical delivery under reordering, loss, equivocation, poisoned tags, and restart. -6. **Certified consensus projection:** add optional consensus vertices, strong parents, explicit - leader choice, contiguous delivery frontiers, and strict clean-only committer consumers. -7. **Frontier linearizer and recovery:** commit deterministic frontier deltas, persist/reconstruct - prefixes and anchors, and add late-node and crash/restart tests. -8. **Benchmarks:** compare the complete protocol with direct `starfish-rbc`, unsafe `starfish-mac`, - signature Starfish variants, and Sailfish++ before attempting tree dissemination. -9. **Tree dissemination:** distribute vector sub-bundles with redundant routing and a direct timeout - fallback; do not change RBC or consensus semantics. - -## 20. Decisions intentionally deferred - -The following values are not safe to guess in the documentation milestone and must be resolved by -the executable model or measured prototype: - -- exact canonical field widths and maximum phase-batch size; -- maximum future-carrier buffer and payload runahead; -- the control-heartbeat rate under low load and backpressure; -- a safe state-retirement, garbage-collection, and late-catch-up watermark; -- whether all supported storage backends are required before authoritative mode; -- quantitative shadow-promotion thresholds and acceptable latency/bandwidth regression; and -- the tree topology, redundancy, and fallback timers. - -Mixed `starfish-rbc`, `starfish-rbc-dag`, and version-one/version-two deployments must be rejected -by protocol-instance negotiation. The provisional `starfish-rbc-dag` selector is added only after -the codec/model milestone establishes a distinct stable version. From 282c280c2955d8e57601e850fd90d050ac222828 Mon Sep 17 00:00:00 2001 From: NaitsabesMue <51112618+NaitsabesMue@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:01:26 +0200 Subject: [PATCH 62/62] remove obsolete standalone RBC-DAG --- README.md | 7 +- crates/orchestrator/README.md | 3 + crates/orchestrator/src/benchmark.rs | 5 +- crates/orchestrator/src/protocol/starfish.rs | 25 +++--- crates/starfish-core/src/block_manager.rs | 4 +- crates/starfish-core/src/broadcaster.rs | 12 +-- crates/starfish-core/src/core.rs | 8 +- crates/starfish-core/src/net_sync.rs | 7 +- crates/starfish-core/src/types.rs | 6 +- crates/starfish-core/src/validator.rs | 13 ++- crates/starfish/src/main.rs | 88 ++------------------ docs/starfish-rbc-single-dag-v3.md | 11 +-- local-dryrun/README.md | 5 +- 13 files changed, 70 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 85d19d07..7d622ff8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository is a benchmarking framework for DAG-based BFT consensus protocols in the partially synchronous model, implemented in Rust. -It includes 10 protocol implementations with configurable +It includes 11 protocol implementations with configurable dissemination strategies, storage backends, and Byzantine fault injection. @@ -23,6 +23,7 @@ injection. | Sparse-Starfish-Speed | `sparse-starfish-speed` | 4.5δ | Uncertified | Encoded | Push | O(n²) | O(n³) | -- | | Starfish | `starfish` | 5.5δ | Uncertified | Encoded | Push | O(n⁴) | O(n⁴) | [eprint.iacr.org/2025/567](https://eprint.iacr.org/2025/567) | | Starfish-RBC (prototype) | `starfish-rbc` | TBD | RBC-certified headers | Encoded | Push | TBD | TBD | [design](docs/starfish-rbc-protocol.md) | +| Single-DAG Starfish-RBC (testbed) | `starfish-rbc-single-dag` | TBD | RBC-certified blocks | Encoded | Push | TBD | TBD | [design](docs/starfish-rbc-single-dag-v3.md) | | Cordial Miners | `cordial-miners` | 6δ | Uncertified | Full | Push | O(n³) | O(n⁴) | [arxiv.org/pdf/2205.09174](https://arxiv.org/pdf/2205.09174) | | Sailfish++ | `sailfish-pp` | 6δ | Certified | Full | Pull | O(n³) | O(n⁴) | [arxiv.org/abs/2505.02761](https://arxiv.org/abs/2505.02761) | | Starfish-BLS | `starfish-bls` | 6.5δ | Uncertified | Encoded | Push | O(n²) | O(n³) | [eprint.iacr.org/2025/567](https://eprint.iacr.org/2025/567)* | @@ -48,6 +49,10 @@ acknowledgment references between validators. headers. ECHO and READY are recipient-authenticated with pairwise MACs; the author's INIT can use Ed25519, ML-DSA-44, ML-DSA-65, or one recipient-specific MAC. It is a correctness-oriented research prototype with the limitations documented in its [protocol specification](docs/starfish-rbc-protocol.md). +**Single-DAG Starfish-RBC** embeds ECHO/READY references in ordinary Starfish +blocks, avoiding a second carrier DAG. Its optional receiver-local quorum-ECHO +flag is a signature-free latency lower bound and does not provide Byzantine +totality; see the [V3 design](docs/starfish-rbc-single-dag-v3.md). **Starfish-Speed** adds strong-vote optimistic sequencing for lower latency when validators share the leader's acknowledgments. **Sparse-Starfish-Speed** (work in progress) combines Bluestreak's diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md index 39afa7b3..33eb82bc 100644 --- a/crates/orchestrator/README.md +++ b/crates/orchestrator/README.md @@ -136,6 +136,9 @@ for any protocol with `--block-authentication ed25519|ml-dsa-44|ml-dsa-65`; Ed25519 is the default. The `starfish-mac`, `starfish-speed-mac`, `sparse-starfish-speed-mac`, and `bluestreak-mac` names are separate experimental protocols and cannot be combined with that option. +The research protocols `starfish-rbc` and `starfish-rbc-single-dag` are also +available; pairwise-MAC initial authentication is selected with +`--block-authentication mac`. To run with Byzantine validators: diff --git a/crates/orchestrator/src/benchmark.rs b/crates/orchestrator/src/benchmark.rs index cec244f1..4f8d84f4 100644 --- a/crates/orchestrator/src/benchmark.rs +++ b/crates/orchestrator/src/benchmark.rs @@ -54,8 +54,9 @@ pub struct BenchmarkParametersGeneric { /// single VPC, they should use their internal IPs to avoid /// paying for data sent between the nodes. pub use_internal_ip_address: bool, - /// Consensus protocol to deploy. The block signature is configured in - /// `node_parameters`; the `*-mac` names denote experimental protocols. + // Consensus protocol to deploy + // (starfish | starfish-speed | sparse-starfish-speed | starfish-bls | + // mysticeti | mysticeti-bls | cordial-miners | bluestreak | sailfish-pp) pub consensus_protocol: String, /// number Byzantine nodes pub byzantine_nodes: usize, diff --git a/crates/orchestrator/src/protocol/starfish.rs b/crates/orchestrator/src/protocol/starfish.rs index 11a7fe72..28630799 100644 --- a/crates/orchestrator/src/protocol/starfish.rs +++ b/crates/orchestrator/src/protocol/starfish.rs @@ -263,7 +263,10 @@ impl StarfishProtocol { consensus_protocol: &str, mut node_parameters: StarfishNodeParameters, ) -> StarfishNodeParameters { - if consensus_protocol == "starfish-rbc" { + if matches!( + consensus_protocol, + "starfish-rbc" | "starfish-rbc-single-dag" + ) { node_parameters.refresh_starfish_rbc_protocol_instance(); } node_parameters @@ -303,15 +306,17 @@ mod tests { #[test] fn starfish_rbc_genesis_gets_one_nonzero_protocol_instance() { - let parameters = StarfishProtocol::node_parameters_for_genesis( - "starfish-rbc", - StarfishNodeParameters::default(), - ); - assert!( - parameters - .starfish_rbc_protocol_instance - .is_some_and(|instance| instance != [0; 32]) - ); + for protocol in ["starfish-rbc", "starfish-rbc-single-dag"] { + let parameters = StarfishProtocol::node_parameters_for_genesis( + protocol, + StarfishNodeParameters::default(), + ); + assert!( + parameters + .starfish_rbc_protocol_instance + .is_some_and(|instance| instance != [0; 32]) + ); + } } #[test] diff --git a/crates/starfish-core/src/block_manager.rs b/crates/starfish-core/src/block_manager.rs index 0cbf079f..484b45eb 100644 --- a/crates/starfish-core/src/block_manager.rs +++ b/crates/starfish-core/src/block_manager.rs @@ -303,7 +303,7 @@ mod tests { .dag_state } - fn open_rbc_dag_state(committee: Arc, path: &std::path::Path) -> DagState { + fn open_starfish_rbc_state(committee: Arc, path: &std::path::Path) -> DagState { let registry = Registry::new(); let (metrics, _reporter) = Metrics::new( ®istry, @@ -421,7 +421,7 @@ mod tests { fn starfish_rbc_requests_ack_only_dependencies_and_deduplicates_parent_overlap() { let committee = Committee::new_for_benchmarks(4); let temp_dir = TempDir::new().unwrap(); - let dag_state = open_rbc_dag_state(committee.clone(), temp_dir.path()); + let dag_state = open_starfish_rbc_state(committee.clone(), temp_dir.path()); let mut manager = BlockManager::new(dag_state, &committee); let genesis: Vec<_> = committee .authorities() diff --git a/crates/starfish-core/src/broadcaster.rs b/crates/starfish-core/src/broadcaster.rs index 2a65ce78..2f2f1a83 100644 --- a/crates/starfish-core/src/broadcaster.rs +++ b/crates/starfish-core/src/broadcaster.rs @@ -1663,8 +1663,8 @@ mod tests { .collect(); let expected = tags[2]; block.header.authentication = BlockAuthentication::MacVector(tags); - let mut rbc_carrier = block.clone(); - rbc_carrier.header.authentication = BlockAuthentication::None; + let mut rbc_header = block.clone(); + rbc_header.header.authentication = BlockAuthentication::None; let relayed = prepare_forwarded_blocks_for_peer( BlockAuthenticationScheme::MacVector, @@ -1686,15 +1686,15 @@ mod tests { ); assert!(second_hop.is_empty()); - let forwarded_rbc_carrier = prepare_forwarded_blocks_for_peer( + let forwarded_rbc_header = prepare_forwarded_blocks_for_peer( BlockAuthenticationScheme::MacVector, ConsensusProtocol::StarfishRbc, 3, - vec![Data::new(rbc_carrier)], + vec![Data::new(rbc_header)], ); - assert_eq!(forwarded_rbc_carrier.len(), 1); + assert_eq!(forwarded_rbc_header.len(), 1); assert!(matches!( - forwarded_rbc_carrier[0].authentication(), + forwarded_rbc_header[0].authentication(), BlockAuthentication::None )); } diff --git a/crates/starfish-core/src/core.rs b/crates/starfish-core/src/core.rs index a6434175..d2fd346d 100644 --- a/crates/starfish-core/src/core.rs +++ b/crates/starfish-core/src/core.rs @@ -2,7 +2,7 @@ // Modifications Copyright (c) 2025 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{collections::BTreeSet, fmt, mem, sync::Arc}; +use std::{collections::BTreeSet, mem, sync::Arc}; use ahash::{AHashMap, AHashSet}; use reed_solomon_simd::ReedSolomonEncoder; @@ -520,7 +520,8 @@ impl Core { let protocol = self.dag_state.consensus_protocol; // Dual-DAG protocols: require clean parent quorum before creating a block. - if protocol.uses_dual_dag() && !protocol.is_starfish_rbc_single_dag() + if protocol.uses_dual_dag() + && !protocol.is_starfish_rbc_single_dag() && clock_round > 1 && !self.dag_state.clean_parent_quorum(clock_round - 1) { @@ -537,7 +538,8 @@ impl Core { // Starfish-RBC that local header is dirty until the local RBC instance // delivers it; another clean quorum must not let us smuggle this dirty // mandatory parent into a proposal. - if protocol.is_starfish_rbc() && !protocol.is_starfish_rbc_single_dag() + if protocol.is_starfish_rbc() + && !protocol.is_starfish_rbc_single_dag() && clock_round > 1 && self .last_own_block diff --git a/crates/starfish-core/src/net_sync.rs b/crates/starfish-core/src/net_sync.rs index 226beaee..665db02a 100644 --- a/crates/starfish-core/src/net_sync.rs +++ b/crates/starfish-core/src/net_sync.rs @@ -46,7 +46,8 @@ use crate::{ shard_reconstructor::{DecodedBlocks, ShardMessage, start_shard_reconstructor}, starfish_rbc::{RbcCanonicalHeader, RbcProtocolInstanceId}, starfish_rbc_service::{ - RbcInitialAuthenticator, RbcServiceEvent, RbcServiceHandle, start_starfish_rbc_service, + RbcInitialAuthenticator, RbcPhaseAuthorityV1, RbcServiceEvent, RbcServiceHandle, + start_starfish_rbc_service_with_phase_authority, }, syncer::{CommitObserver, STARFISH_RBC_SINGLE_DAG_ROUND_INTERVAL, Syncer, SyncerSignals}, types::{ @@ -146,8 +147,8 @@ fn verify_starfish_rbc_transaction_payload( /// specific peer. Legacy MAC-experiment blocks retain their complete vector /// only at direct recipients; forwarding selects the destination's tag. A /// tag-only copy cannot be forwarded again and is therefore omitted. -/// Starfish-RBC carriers are authentication-free and remain forwardable: the -/// separate RBC service, rather than the carrier, controls clean admission. +/// Direct Starfish-RBC header blocks are authentication-free and remain +/// forwardable: the separate RBC service controls clean admission. pub(crate) fn prepare_forwarded_blocks_for_peer( authentication_scheme: BlockAuthenticationScheme, consensus_protocol: ConsensusProtocol, diff --git a/crates/starfish-core/src/types.rs b/crates/starfish-core/src/types.rs index 6edca2ff..568612d1 100644 --- a/crates/starfish-core/src/types.rs +++ b/crates/starfish-core/src/types.rs @@ -3315,7 +3315,7 @@ mod tests { } #[test] - fn starfish_rbc_carrier_uses_canonical_content_identity_without_authentication() { + fn starfish_rbc_header_uses_canonical_content_identity_without_authentication() { let parents = vec![ BlockReference::new_test(0, 1), BlockReference::new_test(1, 1), @@ -3338,7 +3338,7 @@ mod tests { let commitment = block .header() .transactions_commitment - .expect("RBC carrier must commit its Starfish payload"); + .expect("RBC header must commit its Starfish payload"); assert_eq!( block.digest(), BlockDigest::new_starfish_rbc_header( @@ -3402,7 +3402,7 @@ mod tests { } #[test] - fn starfish_rbc_carrier_verification_is_content_only() { + fn starfish_rbc_header_verification_is_content_only() { let committee = Committee::new_for_benchmarks(4); let parents: Vec<_> = committee .authorities() diff --git a/crates/starfish-core/src/validator.rs b/crates/starfish-core/src/validator.rs index 7cc98c6b..ef42a7ae 100644 --- a/crates/starfish-core/src/validator.rs +++ b/crates/starfish-core/src/validator.rs @@ -347,7 +347,7 @@ mod smoke_tests { let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); - if consensus == "starfish-rbc" { + if matches!(consensus, "starfish-rbc" | "starfish-rbc-single-dag") { public_config .parameters .refresh_starfish_rbc_protocol_instance(); @@ -383,7 +383,12 @@ mod smoke_tests { // Four RBC authentication variants run in parallel in the full test // suite and include expensive ML-DSA signing. Give that composed flow // enough scheduling headroom without relaxing existing protocols. - let timeout_multiplier = if consensus == "starfish-rbc" { 20 } else { 5 }; + let timeout_multiplier = if matches!(consensus, "starfish-rbc" | "starfish-rbc-single-dag") + { + 20 + } else { + 5 + }; let timeout = config::param_defaults::default_leader_timeout() * timeout_multiplier; tokio::select! { @@ -429,6 +434,7 @@ mod smoke_tests { #[test_case("starfish-rbc", Some("mac"), 1440)] #[test_case("starfish-rbc", Some("ml-dsa-44"), 1480)] #[test_case("starfish-rbc", Some("ml-dsa-65"), 1520)] + #[test_case("starfish-rbc-single-dag", Some("mac"), 1640)] #[tokio::test] async fn validator_commit( consensus: &str, @@ -485,7 +491,7 @@ mod smoke_tests { let mut public_config = NodePublicConfig::new_for_tests(committee_size).with_port_offset(port_offset); public_config.parameters.block_authentication = block_authentication.map(str::to_string); - if consensus == "starfish-rbc" { + if matches!(consensus, "starfish-rbc" | "starfish-rbc-single-dag") { public_config .parameters .refresh_starfish_rbc_protocol_instance(); @@ -588,6 +594,7 @@ mod smoke_tests { #[test_case("bluestreak", Some("ml-dsa-44"), 980)] #[test_case("bluestreak", Some("ml-dsa-65"), 1260)] #[test_case("starfish-rbc", Some("mac"), 1560)] + #[test_case("starfish-rbc-single-dag", Some("mac"), 1660)] #[tokio::test] async fn validator_sync(consensus: &str, block_authentication: Option<&str>, port_offset: u16) { run_sync_test(consensus, block_authentication, port_offset).await; diff --git a/crates/starfish/src/main.rs b/crates/starfish/src/main.rs index bdafb49d..cdd275ca 100644 --- a/crates/starfish/src/main.rs +++ b/crates/starfish/src/main.rs @@ -416,7 +416,10 @@ async fn local_benchmark( let ips = vec![IpAddr::V4(Ipv4Addr::LOCALHOST); committee_size]; let committee = Committee::new_for_benchmarks(committee_size); load /= committee.len(); - let parameters = Parameters::almost_default(load); + let mut parameters = Parameters::almost_default(load); + // Marks this explicitly bounded local run as a testbed execution. The + // receiver-local quorum-ECHO path is rejected when no finite bound exists. + parameters.benchmark_duration = Some(Duration::from_secs(duration_secs)); // Equivocating Byzantine strategies must not generate transactions. let byzantine_parameters = if ByzantineStrategy::from_strategy_str(&byzantine_strategy) .is_some_and(|s| s.is_equivocating()) @@ -703,75 +706,12 @@ async fn dryrun( Ok(()) } -fn ensure_starfish_rbc_protocol_instance( - consensus_protocol: &str, - node_parameters: &mut NodeParameters, -) { - if is_starfish_rbc_selection(consensus_protocol) - && node_parameters.starfish_rbc_protocol_instance.is_none() - { - node_parameters.refresh_starfish_rbc_protocol_instance(); - } -} - fn is_starfish_rbc_selection(consensus_protocol: &str) -> bool { matches!( consensus_protocol, "starfish-rbc" | "starfish-rbc-single-dag" ) } - -fn validate_local_benchmark_port_offset( - public_config: &NodePublicConfig, - port_offset: u16, -) -> Result<()> { - eyre::ensure!( - public_config.all_network_addresses().all(|address| { - address - .port() - .checked_add(port_offset) - .is_some_and(|port| port <= LOCAL_BENCHMARK_MAX_ACTIVE_BIND_PORT / 10) - }), - "local benchmark port offset {port_offset} places a derived active-bind port in the OS ephemeral range; choose a smaller offset" - ); - Ok(()) -} - -fn preflight_local_benchmark_ports(public_config: &NodePublicConfig) -> Result<()> { - let mut addresses = BTreeSet::new(); - for address in public_config.all_network_addresses() { - addresses.insert(address); - let mut active = address; - active.set_port( - address - .port() - .checked_mul(10) - .expect("validated local benchmark active port"), - ); - addresses.insert(active); - } - addresses.extend(public_config.all_metric_addresses()); - - // The network listeners enable SO_REUSEPORT. A plain bind probe can - // therefore succeed even while a stale benchmark is still serving the - // same address. Probe for an existing listener first, then reserve every - // address for the duration of this check. - for address in &addresses { - eyre::ensure!( - TcpStream::connect_timeout(address, Duration::from_millis(25)).is_err(), - "local benchmark port {address} is already served by another process" - ); - } - - let mut reservations = Vec::with_capacity(addresses.len()); - for address in addresses { - reservations.push( - TcpListener::bind(address) - .wrap_err_with(|| format!("local benchmark port {address} is already in use"))?, - ); - } - Ok(()) -} fn ipv4_add_offset(base: Ipv4Addr, offset: usize) -> Result { let offset = u32::try_from(offset).context("validator count exceeds IPv4 offset range")?; let next = u32::from(base) @@ -886,7 +826,7 @@ mod tests { "--committee-size", "4", "--consensus", - "starfish-rbc", + "starfish-rbc-single-dag", "--block-authentication", "mac", "--starfish-rbc-single-dag-echo-qc-fast-path", @@ -902,24 +842,8 @@ mod tests { else { panic!("expected local-benchmark operation"); }; - assert_eq!(consensus, "starfish-rbc"); + assert_eq!(consensus, "starfish-rbc-single-dag"); assert_eq!(block_authentication.as_deref(), Some("mac")); assert!(starfish_rbc_single_dag_echo_qc_fast_path); } - - #[test] - fn dry_run_starfish_rbc_configuration_gets_a_protocol_instance() { - let mut parameters = NodeParameters { - starfish_rbc_dag_shadow: true, - ..NodeParameters::default() - }; - - ensure_starfish_rbc_protocol_instance("starfish-rbc", &mut parameters); - - assert!( - parameters - .starfish_rbc_protocol_instance - .is_some_and(|instance| instance != [0; 32]) - ); - } } diff --git a/docs/starfish-rbc-single-dag-v3.md b/docs/starfish-rbc-single-dag-v3.md index 9fb03b4f..38fa0b8c 100644 --- a/docs/starfish-rbc-single-dag-v3.md +++ b/docs/starfish-rbc-single-dag-v3.md @@ -2,11 +2,8 @@ ## Status -This is the active experimental successor to the two-plane carrier prototype -documented in `starfish-rbc-dag-protocol.md`. It is selected as -`starfish-rbc-single-dag`. The old implementation remains available as a -benchmark baseline; its carrier and projection formats are frozen and are not -reinterpreted as V3. +This is the active experimental one-block design built on the direct-header +Starfish-RBC implementation. It is selected as `starfish-rbc-single-dag`. V3 is a research-testbed protocol. Crash recovery, bounded retirement and the complete asynchronous safety/liveness proof remain required before production @@ -85,7 +82,7 @@ create a second physical round counter. - RBC delivery never follows from dirty-DAG admission alone. - Consensus parent selection and commitment use only clean vertices. - Recovery content must recompute to the exact requested `BlockReference`. -- Frozen direct-RBC and carrier-DAG formats retain their old domains. +- Direct-RBC formats retain their existing domains. ### Receiver-local quorum-ECHO latency lower bound @@ -121,7 +118,7 @@ conflicting sender locks, dirty-clock progress, clean dependency closure, embedded ECHO/READY delivery, absence of normal phase messages, Byzantine withholding with bounded recovery, restart replay and deterministic Starfish commit order. Matched n=10 and n=40 zero/AWS runs must compare V3 against the -frozen carrier baseline using identical load and duration. +direct-RBC baseline using identical load and duration. ## Testbed checkpoint diff --git a/local-dryrun/README.md b/local-dryrun/README.md index 5b568c2e..2003eccf 100644 --- a/local-dryrun/README.md +++ b/local-dryrun/README.md @@ -35,8 +35,9 @@ NUM_NODES=10 DESIRED_TPS=1000 CONSENSUS=starfish \ Supported `CONSENSUS` values are `starfish`, `starfish-speed`, `sparse-starfish-speed`, `bluestreak`, `starfish-bls`, `cordial-miners`, -`mysticeti`, `sailfish-pp`, and `mysticeti-bls`. `BLOCK_AUTHENTICATION` -selects Ed25519, ML-DSA-44, or ML-DSA-65 for any of them. The +`mysticeti`, `sailfish-pp`, `mysticeti-bls`, `starfish-rbc`, and +`starfish-rbc-single-dag`. `BLOCK_AUTHENTICATION` selects Ed25519, ML-DSA-44, +ML-DSA-65, or `mac` (the latter only for Starfish-RBC) for any of them. The `starfish-mac`, `starfish-speed-mac`, `sparse-starfish-speed-mac`, and `bluestreak-mac` names are separate experimental protocols; leave `BLOCK_AUTHENTICATION` unset when using one.